def _test_no_unneeded_import(self):
import mod_testplug
list(plugin.get_plugins('spork', mod_testplug))
sys.modules.pop('mod_testplug.plug')
# This one is not loaded if we are testing with a good cache.
sys.modules.pop('mod_testplug.plug2', None)
list(plugin.get_plugins('plugtest', mod_testplug))
# Extra messages since getting all of sys.modules printed is annoying.
assert 'mod_testplug.plug' in sys.modules, 'plug not loaded'
assert 'mod_testplug.plug2' not in sys.modules, 'plug2 loaded'
def _test_no_unneeded_import(self):
import mod_testplug
list(plugin.get_plugins('spork', mod_testplug))
sys.modules.pop('mod_testplug.plug')
# This one is not loaded if we are testing with a good cache.
sys.modules.pop('mod_testplug.plug2', None)
list(plugin.get_plugins('plugtest', mod_testplug))
# Extra messages since getting all of sys.modules printed is annoying.
assert 'mod_testplug.plug' in sys.modules, 'plug not loaded'
assert 'mod_testplug.plug2' not in sys.modules, 'plug2 loaded'
def main(f=sys.stdout, **kwargs):
def out(s, **kwargs):
print(s, file=f, **kwargs)
def _rst_header(char, text, newline=True):
if newline:
out('\n', end='')
out(text)
out(char * len(text))
# add module docstring to output doc
if __doc__ is not None:
out(__doc__.strip())
reporters = sorted(unstable_unique(get_plugins('reporter', plugins)),
key=lambda x: x.__name__)
_rst_header('=', 'Reporters', newline=False)
for reporter in reporters:
if reporter.__doc__ is not None:
try:
summary, explanation = reporter.__doc__.split('\n', 1)
except ValueError:
summary = reporter.__doc__
explanation = None
else:
summary = None
out('\n{}'.format(reporter.__name__))
if summary:
out('\t' + ' '.join(dedent(summary).strip().split('\n')))
if explanation:
out('\n\t' +
'\n\t'.join(dedent(explanation).strip().split('\n')))
def test_cache_corruption(self):
print(plugin.const)
print('wheeeeee')
import mod_testplug
list(plugin.get_plugins('spork', mod_testplug))
filename = pjoin(self.packdir, plugin.CACHE_FILENAME)
cachefile = open(filename, 'a')
try:
cachefile.write('corruption\n')
finally:
cachefile.close()
# Shift the file into the past a little or the rewritten file
# will occasionally have the same mtime as the corrupt one.
st = os.stat(filename)
corrupt_mtime = st.st_mtime - 2
os.utime(filename, (st.st_atime, corrupt_mtime))
plugin._global_cache.clear()
self._test_plug()
good_mtime = os.path.getmtime(
pjoin(self.packdir, plugin.CACHE_FILENAME))
plugin._global_cache.clear()
self._test_plug()
assert good_mtime == os.path.getmtime(
pjoin(self.packdir, plugin.CACHE_FILENAME))
assert good_mtime != corrupt_mtime
def load_config(user_conf_file=USER_CONF_FILE,
system_conf_file=SYSTEM_CONF_FILE,
debug=False, prepend_sources=(), append_sources=(), skip_config_files=False):
"""
the main entry point for any code looking to use pkgcore.
:param user_conf_file: file to attempt to load, else defaults to trying to
load portage 2 style configs (/etc/make.conf, /etc/make.profile)
:return: :obj:`pkgcore.config.central.ConfigManager` instance
representing the system config.
"""
from pkgcore.config import central, cparser
from pkgcore.plugin import get_plugins
import os
configs = list(prepend_sources)
configs.extend(get_plugins('global_config'))
if not skip_config_files:
have_system_conf = os.path.isfile(system_conf_file)
have_user_conf = os.path.isfile(user_conf_file)
if have_system_conf or have_user_conf:
if have_system_conf:
configs.append(
cparser.config_from_file(open(system_conf_file)))
if have_user_conf:
configs.append(cparser.config_from_file(open(user_conf_file)))
else:
# make.conf...
from pkgcore.ebuild.portage_conf import config_from_make_conf
configs.append(config_from_make_conf())
configs.extend(append_sources)
return central.CompatConfigManager(central.ConfigManager(configs, debug=debug))
def __init__(self,
mode,
tempdir,
hooks,
csets,
preserves,
observer,
offset=None,
disable_plugins=False,
parallelism=None):
if observer is None:
observer = observer_mod.repo_observer(observer_mod.null_output)
self.observer = observer
self.mode = mode
if tempdir is not None:
tempdir = normpath(tempdir) + '/'
self.tempdir = tempdir
self.parallelism = parallelism if parallelism is not None else cpu_count(
)
self.hooks = ImmutableDict((x, []) for x in hooks)
self.preserve_csets = []
self.cset_sources = {}
# instantiate these separately so their values are preserved
self.preserved_csets = LazyValDict(self.preserve_csets,
self._get_cset_source)
for k, v in csets.items():
if isinstance(v, str):
v = getattr(self, v, v)
if not callable(v):
raise TypeError(
"cset values must be either the string name of "
f"existing methods, or callables (got {v})")
if k in preserves:
self.add_preserved_cset(k, v)
else:
self.add_cset(k, v)
if offset is None:
offset = "/"
self.offset = offset
if not disable_plugins:
# merge in default triggers first.
for trigger in get_plugins('triggers'):
t = trigger()
t.register(self)
# merge in overrides
for hook, triggers in hooks.items():
for trigger in triggers:
self.add_trigger(hook, trigger)
self.regenerate_csets()
for x in hooks:
setattr(self, x, partial(self.execute_hook, x))
def __init__(self,
suppress=False,
config=True,
domain=True,
script=None,
**kwds):
super().__init__(suppress=suppress, script=script, **kwds)
self.register('action', 'parsers', _SubParser)
if not suppress:
config_opts = self.add_argument_group("config options")
if config:
config_opts.add_argument(
'--add-config',
nargs=3,
action='append',
metavar=('SECTION', 'KEY', 'VALUE'),
help='modify an existing configuration section')
config_opts.add_argument(
'--new-config',
nargs=3,
action='append',
metavar=('SECTION', 'KEY', 'VALUE'),
help='add a new configuration section')
config_opts.add_argument(
'--empty-config',
action='store_true',
help='do not load user/system configuration')
config_opts.add_argument(
'--config',
metavar='PATH',
dest='override_config',
type=arghparse.existent_path,
help='override location of config files')
if script is not None:
try:
_, script_module = script
except TypeError:
raise ValueError(
"invalid script parameter, should be (__file__, __name__)"
)
project = script_module.split('.')[0]
else:
project = __name__.split('.')[0]
# TODO: figure out a better method for plugin registry/loading
try:
plugins = import_module('.plugins', project)
global_config = get_plugins('global_config', plugins)
self.set_defaults(config=arghparse.DelayedValue(
partial(store_config, global_config=global_config)))
except ImportError:
pass
if domain:
_mk_domain(config_opts)
def GenericSyncer(basedir, uri, **kwargs):
"""Syncer using the plugin system to find a syncer based on uri."""
plugins = list((plug.supports_uri(uri), plug)
for plug in plugin.get_plugins('syncer'))
plugins.sort(key=lambda x: x[0])
if not plugins or plugins[-1][0] <= 0:
raise UriError(uri, "no known syncer support")
# XXX this is random if there is a tie. Should we raise an exception?
return plugins[-1][1](basedir, uri, **kwargs)
def _replay_validate_args(parser, namespace):
func = namespace.config.pkgcheck_reporter_factory.get(namespace.reporter)
if func is None:
func = list(
base.Whitelist([namespace.reporter
]).filter(get_plugins('reporter', plugins)))
if not func:
available = ', '.join(
sorted(x.__name__ for x in get_plugins('reporter', plugins)))
parser.error(f"no reporter matches {namespace.reporter!r} "
f"(available: {available})")
elif len(func) > 1:
reporters = tuple(
sorted(f"{x.__module__}.{x.__name__}" for x in func))
parser.error(
f"reporter {namespace.reporter!r} matched multiple reporters, "
f"must match one. {reporters!r}")
func = func[0]
namespace.reporter = func
def __init__(self, mode, tempdir, hooks, csets, preserves, observer,
offset=None, disable_plugins=False, parallelism=None):
if observer is None:
observer = observer_mod.repo_observer(observer_mod.null_output)
self.observer = observer
self.mode = mode
if tempdir is not None:
tempdir = normpath(tempdir) + '/'
self.tempdir = tempdir
if parallelism is None:
parallelism = get_proc_count()
self.parallelism = parallelism
self.hooks = ImmutableDict((x, []) for x in hooks)
self.preserve_csets = []
self.cset_sources = {}
# instantiate these separately so their values are preserved
self.preserved_csets = LazyValDict(
self.preserve_csets, self._get_cset_source)
for k, v in csets.iteritems():
if isinstance(v, basestring):
v = getattr(self, v, v)
if not callable(v):
raise TypeError(
"cset values must be either the string name of "
"existing methods, or callables (got %s)" % v)
if k in preserves:
self.add_preserved_cset(k, v)
else:
self.add_cset(k, v)
if offset is None:
offset = "/"
self.offset = offset
if not disable_plugins:
# merge in default triggers first.
for trigger in get_plugins('triggers'):
t = trigger()
t.register(self)
# merge in overrides
for hook, triggers in hooks.iteritems():
for trigger in triggers:
self.add_trigger(hook, trigger)
self.regenerate_csets()
for x in hooks:
setattr(self, x, partial(self.execute_hook, x))
def _show(options, out, err):
if options.keywords:
display_keywords(out, options)
if options.checks:
display_checks(out, options)
if options.reporters:
display_reporters(
out, options,
list(options.config.pkgcheck_reporter_factory.values()),
list(get_plugins('reporter', plugins)))
return 0
def load_config(user_conf_file=USER_CONF_FILE,
system_conf_file=SYSTEM_CONF_FILE,
debug=False,
prepend_sources=(),
append_sources=(),
skip_config_files=False,
profile_override=None,
location='/etc/'):
"""
the main entry point for any code looking to use pkgcore.
:param user_conf_file: file to attempt to load, else defaults to trying to
load portage 2 style configs (/etc/portage/make.conf and
/etc/portage/make.profile or the deprecated /etc/make.conf and
/etc/make.profile locations)
:param location: location the portage configuration is based in,
defaults to /etc
:param profile_override: profile to use instead of the current system
profile, i.e. the target path of the /etc/portage/make.profile
(or deprecated /etc/make.profile) symlink
:return: :obj:`pkgcore.config.central.ConfigManager` instance
representing the system config.
"""
from pkgcore.config import central, cparser
from pkgcore.plugin import get_plugins
import os
configs = list(prepend_sources)
configs.extend(get_plugins('global_config'))
if not skip_config_files:
have_system_conf = os.path.isfile(system_conf_file)
have_user_conf = os.path.isfile(user_conf_file)
if have_system_conf or have_user_conf:
if have_system_conf:
with open(system_conf_file) as f:
configs.append(cparser.config_from_file(f))
if have_user_conf:
with open(user_conf_file) as f:
configs.append(cparser.config_from_file(f))
else:
# make.conf...
from pkgcore.ebuild.portage_conf import config_from_make_conf
configs.append(
config_from_make_conf(location=location,
profile_override=profile_override))
configs.extend(append_sources)
return central.CompatConfigManager(
central.ConfigManager(configs, debug=debug))
def configurables_main(options, out, err):
"""List registered configurables."""
# try and sort this beast.
def key_func(obj):
return "%s.%s" % (getattr(obj, "__module__", ""), getattr(obj, "__name__", ""))
for configurable in sorted(get_plugins("configurable"), key=key_func):
type_obj = basics.ConfigType(configurable)
if options.typename is not None and type_obj.name != options.typename:
continue
out.write(out.bold, "%s.%s" % (configurable.__module__, configurable.__name__))
write_type(out, type_obj)
out.write()
out.write()
def test_header_change_invalidates_cache(self):
# Write the cache
plugin._global_cache.clear()
import mod_testplug
list(plugin.get_plugins('testplug', mod_testplug))
# Modify the cache.
filename = os.path.join(self.packdir, plugin.CACHE_FILENAME)
cache = list(open(filename))
cache[0] = 'not really a pkgcore plugin cache\n'
open(filename, 'w').write(''.join(cache))
# And test if it is properly rewritten.
plugin._global_cache.clear()
self._test_plug()
def test_rewrite_on_remove(self):
filename = pjoin(self.packdir, 'extra.py')
plug = open(filename, 'w')
try:
plug.write('pkgcore_plugins = {"plugtest": [object()]}\n')
finally:
plug.close()
plugin._global_cache.clear()
import mod_testplug
assert len(list(plugin.get_plugins('plugtest', mod_testplug))) == 3
os.unlink(filename)
plugin._global_cache.clear()
self._test_plug()
def test_rewrite_on_remove(self):
filename = pjoin(self.packdir, 'extra.py')
plug = open(filename, 'w')
try:
plug.write('pkgcore_plugins = {"plugtest": [object()]}\n')
finally:
plug.close()
plugin._global_cache.clear()
import mod_testplug
assert len(list(plugin.get_plugins('plugtest', mod_testplug))) == 3
os.unlink(filename)
plugin._global_cache.clear()
self._test_plug()
def configurables_main(options, out, err):
"""List registered configurables."""
# try and sort this beast.
def key_func(obj):
return "%s.%s" % (getattr(obj, '__module__', ''),
getattr(obj, '__name__', ''))
for configurable in sorted(get_plugins('configurable'), key=key_func):
type_obj = basics.ConfigType(configurable)
if options.typename is not None and type_obj.name != options.typename:
continue
out.write(out.bold, f'{configurable.__module__}.{configurable.__name__}')
write_type(out, type_obj)
out.write()
out.write()
def test_header_change_invalidates_cache(self):
# Write the cache
plugin._global_cache.clear()
import mod_testplug
list(plugin.get_plugins('testplug', mod_testplug))
# Modify the cache.
filename = pjoin(self.packdir, plugin.CACHE_FILENAME)
with open(filename) as f:
cache = f.readlines()
cache[0] = 'not really a pkgcore plugin cache\n'
with open(filename, 'w') as f:
f.write(''.join(cache))
# And test if it is properly rewritten.
plugin._global_cache.clear()
self._test_plug()
def load_config(user_conf_file=USER_CONF_FILE,
system_conf_file=SYSTEM_CONF_FILE,
debug=False, prepend_sources=(), append_sources=(),
skip_config_files=False, profile_override=None,
location='/etc/'):
"""
the main entry point for any code looking to use pkgcore.
:param user_conf_file: file to attempt to load, else defaults to trying to
load portage 2 style configs (/etc/portage/make.conf and
/etc/portage/make.profile or the deprecated /etc/make.conf and
/etc/make.profile locations)
:param location: location the portage configuration is based in,
defaults to /etc
:param profile_override: profile to use instead of the current system
profile, i.e. the target path of the /etc/portage/make.profile
(or deprecated /etc/make.profile) symlink
:return: :obj:`pkgcore.config.central.ConfigManager` instance
representing the system config.
"""
from pkgcore.config import central, cparser
from pkgcore.plugin import get_plugins
import os
configs = list(prepend_sources)
configs.extend(get_plugins('global_config'))
if not skip_config_files:
have_system_conf = os.path.isfile(system_conf_file)
have_user_conf = os.path.isfile(user_conf_file)
if have_system_conf or have_user_conf:
if have_system_conf:
with open(system_conf_file) as f:
configs.append(cparser.config_from_file(f))
if have_user_conf:
with open(user_conf_file) as f:
configs.append(cparser.config_from_file(f))
else:
# make.conf...
from pkgcore.ebuild.portage_conf import config_from_make_conf
configs.append(config_from_make_conf(
location=location, profile_override=profile_override))
configs.extend(append_sources)
return central.CompatConfigManager(central.ConfigManager(configs, debug=debug))
def _test_priority_caching(self):
import mod_testplug
list(plugin.get_plugins('spork', mod_testplug))
sys.modules.pop('mod_testplug.plug', None)
sys.modules.pop('mod_testplug.plug2', None)
sys.modules.pop('mod_testplug.plug3', None)
sys.modules.pop('mod_testplug.plug4', None)
sys.modules.pop('mod_testplug.plug5', None)
sys.modules.pop('mod_testplug.plug6', None)
best_plug = plugin.get_plugin('plugtest', mod_testplug)
from mod_testplug import plug
assert plug.high_plug == best_plug
# Extra messages since getting all of sys.modules printed is annoying.
assert 'mod_testplug.plug' in sys.modules, 'plug not loaded'
assert 'mod_testplug.plug2' not in sys.modules, 'plug2 loaded'
assert 'mod_testplug.plug3' not in sys.modules, 'plug3 loaded'
assert 'mod_testplug.plug4' in sys.modules, 'plug4 not loaded'
assert 'mod_testplug.plug5' in sys.modules, 'plug4 not loaded'
assert 'mod_testplug.plug6' not in sys.modules, 'plug6 loaded'
def _test_priority_caching(self):
import mod_testplug
list(plugin.get_plugins('spork', mod_testplug))
sys.modules.pop('mod_testplug.plug', None)
sys.modules.pop('mod_testplug.plug2', None)
sys.modules.pop('mod_testplug.plug3', None)
sys.modules.pop('mod_testplug.plug4', None)
sys.modules.pop('mod_testplug.plug5', None)
sys.modules.pop('mod_testplug.plug6', None)
best_plug = plugin.get_plugin('plugtest', mod_testplug)
from mod_testplug import plug
assert plug.high_plug == best_plug
# Extra messages since getting all of sys.modules printed is annoying.
assert 'mod_testplug.plug' in sys.modules, 'plug not loaded'
assert 'mod_testplug.plug2' not in sys.modules, 'plug2 loaded'
assert 'mod_testplug.plug3' not in sys.modules, 'plug3 loaded'
assert 'mod_testplug.plug4' in sys.modules, 'plug4 not loaded'
assert 'mod_testplug.plug5' in sys.modules, 'plug4 not loaded'
assert 'mod_testplug.plug6' not in sys.modules, 'plug6 loaded'
def _test_plug(self):
import mod_testplug
assert plugin.get_plugin('spork', mod_testplug) is None
plugins = list(plugin.get_plugins('plugtest', mod_testplug))
assert len(plugins) == 2, plugins
plugin.get_plugin('plugtest', mod_testplug)
assert 'HighPlug' == \
plugin.get_plugin('plugtest', mod_testplug).__class__.__name__
with open(pjoin(self.packdir, plugin.CACHE_FILENAME)) as f:
lines = f.readlines()
assert len(lines) == 3
assert plugin.CACHE_HEADER + "\n" == lines[0]
lines.pop(0)
lines.sort()
mtime = int(os.path.getmtime(pjoin(self.packdir, 'plug2.py')))
assert f'plug2:{mtime}:\n' == lines[0]
mtime = int(os.path.getmtime(pjoin(self.packdir, 'plug.py')))
assert (
f'plug:{mtime}:plugtest,7,1:plugtest,1,tests.test_plugin.LowPlug:plugtest,0,0\n'
== lines[1])
def _test_plug(self):
import mod_testplug
assert plugin.get_plugin('spork', mod_testplug) is None
plugins = list(plugin.get_plugins('plugtest', mod_testplug))
assert len(plugins) == 2, plugins
plugin.get_plugin('plugtest', mod_testplug)
assert 'HighPlug' == \
plugin.get_plugin('plugtest', mod_testplug).__class__.__name__
with open(pjoin(self.packdir, plugin.CACHE_FILENAME)) as f:
lines = f.readlines()
assert len(lines) == 3
assert plugin.CACHE_HEADER + "\n" == lines[0]
lines.pop(0)
lines.sort()
mtime = int(os.path.getmtime(pjoin(self.packdir, 'plug2.py')))
assert f'plug2:{mtime}:\n' == lines[0]
mtime = int(os.path.getmtime(pjoin(self.packdir, 'plug.py')))
assert (
f'plug:{mtime}:plugtest,7,1:plugtest,1,tests.test_plugin.LowPlug:plugtest,0,0\n'
== lines[1])
def _test_plug(self):
import mod_testplug
self.assertIdentical(None, plugin.get_plugin('spork', mod_testplug))
plugins = list(plugin.get_plugins('plugtest', mod_testplug))
self.assertEqual(2, len(plugins), plugins)
plugin.get_plugin('plugtest', mod_testplug)
self.assertEqual(
'HighPlug',
plugin.get_plugin('plugtest', mod_testplug).__class__.__name__)
with open(pjoin(self.packdir, plugin.CACHE_FILENAME)) as f:
lines = f.readlines()
self.assertEqual(3, len(lines))
self.assertEqual(plugin.CACHE_HEADER + "\n", lines[0])
lines.pop(0)
lines.sort()
mtime = int(os.path.getmtime(pjoin(self.packdir, 'plug2.py')))
self.assertEqual('plug2:%s:\n' % (mtime, ), lines[0])
mtime = int(os.path.getmtime(pjoin(self.packdir, 'plug.py')))
self.assertEqual(
'plug:%s:plugtest,7,1:plugtest,1,pkgcore.test.test_plugin.LowPlug:plugtest,0,0\n'
% (mtime, ), lines[1])
def _test_plug(self):
import mod_testplug
self.assertIdentical(None, plugin.get_plugin('spork', mod_testplug))
plugins = list(plugin.get_plugins('plugtest', mod_testplug))
self.assertEqual(2, len(plugins), plugins)
plugin.get_plugin('plugtest', mod_testplug)
self.assertEqual(
'HighPlug',
plugin.get_plugin('plugtest', mod_testplug).__class__.__name__)
lines = list(open(os.path.join(self.packdir, plugin.CACHE_FILENAME)))
self.assertEqual(3, len(lines))
self.assertEqual(plugin.CACHE_HEADER + "\n", lines[0])
lines.pop(0)
lines.sort()
mtime = int(os.path.getmtime(os.path.join(self.packdir, 'plug2.py')))
self.assertEqual('plug2:%s:\n' % (mtime,), lines[0])
mtime = int(os.path.getmtime(os.path.join(self.packdir, 'plug.py')))
self.assertEqual(
'plug:%s:plugtest,7,1:plugtest,0,pkgcore.test.test_plugin.LowPlug:plugtest,0,0\n'
% (mtime,),
lines[1])
def test_cache_corruption(self):
import mod_testplug
list(plugin.get_plugins('spork', mod_testplug))
filename = pjoin(self.packdir, plugin.CACHE_FILENAME)
cachefile = open(filename, 'a')
try:
cachefile.write('corruption\n')
finally:
cachefile.close()
# Shift the file into the past a little or the rewritten file
# will occasionally have the same mtime as the corrupt one.
st = os.stat(filename)
corrupt_mtime = st.st_mtime - 2
os.utime(filename, (st.st_atime, corrupt_mtime))
plugin._global_cache.clear()
self._test_plug()
good_mtime = os.path.getmtime(
pjoin(self.packdir, plugin.CACHE_FILENAME))
plugin._global_cache.clear()
self._test_plug()
self.assertEqual(good_mtime, os.path.getmtime(
pjoin(self.packdir, plugin.CACHE_FILENAME)))
self.assertNotEqual(good_mtime, corrupt_mtime)
def main(options, out, err):
"""Do stuff."""
if options.list_checks:
display_checks(out, options.checks)
return 0
if options.list_reporters:
display_reporters(out, options.config,
options.config.pcheck_reporter_factory.values(),
list(get_plugins('reporter', plugins)))
return 0
if not options.repo_bases:
err.write(
'Warning: could not determine repository base for profiles. '
'Some checks will not work. Either specify a plain target repo '
'(not combined trees) or specify a PORTDIR repo '
'with --overlayed-repo.', wrap=True)
err.write()
if options.guessed_suite:
if options.default_suite:
err.write('Tried to guess a suite to use but got multiple matches')
err.write('and fell back to the default.')
else:
err.write('using suite guessed from working directory')
if options.guessed_target_repo:
err.write('using repository guessed from working directory')
try:
reporter = options.reporter(out)
except errors.ReporterInitError, e:
err.write(err.fg('red'), err.bold, '!!! ', err.reset,
'Error initializing reporter: ', e)
return 1
def check_args(parser, namespace):
# XXX hack...
namespace.checks = sorted(unstable_unique(
get_plugins('check', plugins)),
key=lambda x: x.__name__)
if any((namespace.list_keywords, namespace.list_checks, namespace.list_reporters)):
# no need to check any other args
return
cwd = abspath(os.getcwd())
if namespace.suite is None:
# No suite explicitly specified. Use the repo to guess the suite.
if namespace.target_repo is None:
# Not specified either. Try to find a repo our cwd is in.
# The use of a dict here is a hack to deal with one
# repo having multiple names in the configuration.
candidates = {}
for name, suite in namespace.config.pkgcheck_suite.iteritems():
repo = suite.target_repo
if repo is None:
continue
repo_base = getattr(repo, 'location', None)
if repo_base is not None and cwd.startswith(repo_base):
candidates[repo] = name
if len(candidates) == 1:
namespace.guessed_suite = True
namespace.target_repo = tuple(candidates)[0]
if namespace.target_repo is not None:
# We have a repo, now find a suite matching it.
candidates = list(
suite for suite in namespace.config.pkgcheck_suite.itervalues()
if suite.target_repo is namespace.target_repo)
if len(candidates) == 1:
namespace.guessed_suite = True
namespace.suite = candidates[0]
if namespace.suite is None:
# If we have multiple candidates or no candidates we
# fall back to the default suite.
namespace.suite = namespace.config.get_default('pkgcheck_suite')
namespace.default_suite = namespace.suite is not None
if namespace.suite is not None:
# We have a suite. Lift defaults from it for values that
# were not set explicitly:
if namespace.checkset is None:
namespace.checkset = namespace.suite.checkset
# If we were called with no atoms we want to force
# cwd-based detection.
if namespace.target_repo is None:
if namespace.targets:
namespace.target_repo = namespace.suite.target_repo
elif namespace.suite.target_repo is not None:
# No atoms were passed in, so we want to guess
# what to scan based on cwd below. That only makes
# sense if we are inside the target repo. We still
# want to pick the suite's target repo if we are
# inside it, in case there is more than one repo
# definition with a base that contains our dir.
repo_base = getattr(namespace.suite.target_repo, 'location', None)
if repo_base is not None and cwd.startswith(repo_base):
namespace.target_repo = namespace.suite.target_repo
if namespace.target_repo is None:
# We have no target repo (not explicitly passed, not from a suite, not
# from an earlier guess at the target_repo) so try to guess one.
if len(namespace.targets) == 1 and os.path.exists(namespace.targets[0]):
target_dir = namespace.targets[0]
else:
target_dir = cwd
target_repo = None
for name, repo in namespace.config.repo.iteritems():
repo_base = getattr(repo, 'location', None)
if repo_base is not None and target_dir in repo:
target_repo = repo
if target_repo is None:
parser.error(
'no target repo specified and '
'current directory is not inside a known repo')
namespace.target_repo = target_repo
if namespace.reporter is None:
namespace.reporter = namespace.config.get_default(
'pkgcheck_reporter_factory')
if namespace.reporter is None:
namespace.reporter = get_plugin('reporter', plugins)
if namespace.reporter is None:
parser.error(
'no config defined reporter found, nor any default '
'plugin based reporters')
else:
func = namespace.config.pkgcheck_reporter_factory.get(namespace.reporter)
if func is None:
func = list(base.Whitelist([namespace.reporter]).filter(
get_plugins('reporter', plugins)))
if not func:
parser.error(
"no reporter matches %r (available: %s)" % (
namespace.reporter,
', '.join(sorted(x.__name__ for x in get_plugins('reporter', plugins)))
)
)
elif len(func) > 1:
parser.error(
"--reporter %r matched multiple reporters, "
"must match one. %r" % (
namespace.reporter,
tuple(sorted("%s.%s" % (x.__module__, x.__name__) for x in func))
)
)
func = func[0]
namespace.reporter = func
# search_repo is a multiplex of target_repo and its masters, make sure
# they're configured properly in metadata/layout.conf. This is used for
# things like visibility checks (it is passed to the checkers in "start").
namespace.search_repo = multiplex.tree(*namespace.target_repo.trees)
namespace.repo_bases = [abspath(repo.location) for repo in reversed(namespace.target_repo.trees)]
if namespace.targets:
limiters = []
repo = namespace.target_repo
# read targets from stdin
if len(namespace.targets) == 1 and namespace.targets[0] == '-':
namespace.targets = [x.strip() for x in sys.stdin.readlines() if x.strip() != '']
# reassign stdin to allow interactivity (currently only works for unix)
sys.stdin = open('/dev/tty')
for target in namespace.targets:
try:
limiters.append(parserestrict.parse_match(target))
except parserestrict.ParseError as e:
if os.path.exists(target):
try:
limiters.append(repo.path_restrict(target))
except ValueError as e:
parser.error(e)
else:
parser.error(e)
namespace.limiters = limiters
else:
repo_base = getattr(namespace.target_repo, 'location', None)
if not repo_base:
parser.error(
'Either specify a target repo that is not multi-tree or '
'one or more extended atoms to scan '
'("*" for the entire repo).')
if cwd not in namespace.target_repo:
namespace.limiters = [packages.AlwaysTrue]
else:
namespace.limiters = [packages.AndRestriction(*namespace.target_repo.path_restrict(cwd))]
if namespace.checkset is None:
namespace.checkset = namespace.config.get_default('pkgcheck_checkset')
if namespace.checkset is not None:
namespace.checks = list(namespace.checkset.filter(namespace.checks))
disabled_checks, enabled_checks = ((), ())
if namespace.selected_checks is not None:
disabled_checks, enabled_checks = namespace.selected_checks
if enabled_checks:
whitelist = base.Whitelist(enabled_checks)
namespace.checks = list(whitelist.filter(namespace.checks))
if disabled_checks:
blacklist = base.Blacklist(disabled_checks)
namespace.checks = list(blacklist.filter(namespace.checks))
if not namespace.checks:
parser.error('no active checks')
namespace.addons = set()
def add_addon(addon):
if addon not in namespace.addons:
namespace.addons.add(addon)
for dep in addon.required_addons:
add_addon(dep)
for check in namespace.checks:
add_addon(check)
try:
for addon in namespace.addons:
addon.check_args(parser, namespace)
except argparse.ArgumentError as e:
if namespace.debug:
raise
parser.error(str(e))
def main(options, out, err):
"""Do stuff."""
if options.list_keywords:
display_keywords(out, options.checks)
return 0
if options.list_checks:
display_checks(out, options.checks)
return 0
if options.list_reporters:
display_reporters(
out, options.config,
options.config.pkgcheck_reporter_factory.values(),
list(get_plugins('reporter', plugins)))
return 0
if not options.repo_bases:
err.write(
'Warning: could not determine repo base for profiles, some checks will not work.')
err.write()
if options.guessed_suite:
if options.default_suite:
err.write('Tried to guess a suite to use but got multiple matches')
err.write('and fell back to the default.')
else:
err.write('using suite guessed from working directory')
if options.guessed_target_repo:
err.write('using repository guessed from working directory')
try:
reporter = options.reporter(out)
except errors.ReporterInitError as e:
err.write(
err.fg('red'), err.bold, '!!! ', err.reset,
'Error initializing reporter: ', e)
return 1
addons_map = {}
def init_addon(klass):
res = addons_map.get(klass)
if res is not None:
return res
deps = list(init_addon(dep) for dep in klass.required_addons)
try:
res = addons_map[klass] = klass(options, *deps)
except KeyboardInterrupt:
raise
except Exception:
err.write('instantiating %s' % (klass,))
raise
return res
for addon in options.addons:
# Ignore the return value, we just need to populate addons_map.
init_addon(addon)
if options.debug:
err.write('target repo: ', repr(options.target_repo))
err.write('base dirs: ', repr(options.repo_bases))
for filterer in options.limiters:
err.write('limiter: ', repr(filterer))
debug = logging.debug
else:
debug = None
transforms = list(get_plugins('transform', plugins))
# XXX this is pretty horrible.
sinks = list(addon for addon in addons_map.itervalues()
if getattr(addon, 'feed_type', False))
reporter.start()
for filterer in options.limiters:
sources = [feeds.RestrictedRepoSource(options.target_repo, filterer)]
bad_sinks, pipes = base.plug(sinks, transforms, sources, debug)
if bad_sinks:
# We want to report the ones that would work if this was a
# full repo scan separately from the ones that are
# actually missing transforms.
bad_sinks = set(bad_sinks)
full_scope = feeds.RestrictedRepoSource(
options.target_repo, packages.AlwaysTrue)
really_bad, ignored = base.plug(sinks, transforms, [full_scope])
really_bad = set(really_bad)
assert bad_sinks >= really_bad, \
'%r unreachable with no limiters but reachable with?' % (
really_bad - bad_sinks,)
for sink in really_bad:
err.error(
'sink %s could not be connected (missing transforms?)' % (
sink,))
out_of_scope = bad_sinks - really_bad
if options.verbose and out_of_scope:
err.warn('skipping repo checks (not a full repo scan)')
if not pipes:
out.write(out.fg('red'), ' * ', out.reset, 'No checks!')
else:
if options.debug:
err.write('Running %i tests' % (len(sinks) - len(bad_sinks),))
for source, pipe in pipes:
pipe.start()
reporter.start_check(
list(base.collect_checks_classes(pipe)), filterer)
for thing in source.feed():
pipe.feed(thing, reporter)
pipe.finish(reporter)
reporter.end_check()
reporter.finish()
# flush stdout first; if they're directing it all to a file, this makes
# results not get the final message shoved in midway
out.stream.flush()
return 0
help='limit checks to regex or package/class matching')
check_options.add_argument(
'-C', '--checkset', metavar='CHECKSET', action=commandline.StoreConfigObject,
config_type='pkgcheck_checkset',
help='preconfigured set of checks to run')
all_addons = set()
def add_addon(addon):
if addon not in all_addons:
all_addons.add(addon)
for dep in addon.required_addons:
add_addon(dep)
argparser.plugin = argparser.add_argument_group('plugin options')
for check in get_plugins('check', plugins):
add_addon(check)
for addon in all_addons:
addon.mangle_argparser(argparser)
@argparser.bind_final_check
def check_args(parser, namespace):
# XXX hack...
namespace.checks = sorted(unstable_unique(
get_plugins('check', plugins)),
key=lambda x: x.__name__)
if any((namespace.list_keywords, namespace.list_checks, namespace.list_reporters)):
# no need to check any other args
return
def _scan(options, out, err):
if not options.repo_bases:
err.write(
'Warning: could not determine repo base for profiles, some checks will not work.'
)
err.write()
if options.guessed_suite:
if options.default_suite:
err.write('Tried to guess a suite to use but got multiple matches')
err.write('and fell back to the default.')
else:
err.write('using suite guessed from working directory')
try:
reporter = options.reporter(out,
keywords=options.filtered_keywords,
verbosity=options.verbosity)
except errors.ReporterInitError as e:
err.write(f'{scan.prog}: failed initializing reporter: {e}')
return 1
addons_map = {}
def init_addon(klass):
res = addons_map.get(klass)
if res is not None:
return res
deps = list(init_addon(dep) for dep in klass.required_addons)
res = addons_map[klass] = klass(options, *deps)
return res
for addon in options.addons:
# Ignore the return value, we just need to populate addons_map.
init_addon(addon)
if options.verbosity > 1:
err.write(f"target repo: {options.target_repo.repo_id!r} "
f"at {options.target_repo.location!r}")
err.write('base dirs: ', ', '.join(options.repo_bases))
for filterer in options.limiters:
err.write('limiter: ', filterer)
debug = logging.debug
else:
debug = None
transforms = list(get_plugins('transform', plugins))
# XXX this is pretty horrible.
sinks = list(addon for addon in addons_map.values()
if getattr(addon, 'feed_type', False))
reporter.start()
for filterer in options.limiters:
sources = [feeds.RestrictedRepoSource(options.target_repo, filterer)]
bad_sinks, pipes = base.plug(sinks, transforms, sources, debug)
if bad_sinks:
# We want to report the ones that would work if this was a
# full repo scan separately from the ones that are
# actually missing transforms.
bad_sinks = set(bad_sinks)
full_scope = feeds.RestrictedRepoSource(options.target_repo,
packages.AlwaysTrue)
really_bad, ignored = base.plug(sinks, transforms, [full_scope])
really_bad = set(really_bad)
assert bad_sinks >= really_bad, \
f'{really_bad - bad_sinks} unreachable with no limiters but reachable with?'
for sink in really_bad:
err.error(
f'sink {sink} could not be connected (missing transforms?)'
)
out_of_scope = bad_sinks - really_bad
if options.verbosity > 1 and out_of_scope:
err.warn('skipping repo checks (not a full repo scan)')
if pipes:
if options.debug:
err.write(f'Running {len(sinks) - len(bad_sinks)} tests')
err.flush()
for source, pipe in pipes:
pipe.start(reporter)
reporter.start_check(list(base.collect_checks_classes(pipe)),
filterer)
for thing in source.feed():
pipe.feed(thing, reporter)
pipe.finish(reporter)
reporter.end_check()
else:
err.write(
f'{scan.prog}: no matching checks available for current scope')
reporter.finish()
# flush stdout first; if they're directing it all to a file, this makes
# results not get the final message shoved in midway
out.stream.flush()
return 0
action=commandline.StoreConfigObject,
config_type='pkgcheck_checkset',
help='preconfigured set of checks to run')
all_addons = set()
def add_addon(addon):
if addon not in all_addons:
all_addons.add(addon)
for dep in addon.required_addons:
add_addon(dep)
argparser.plugin = argparser.add_argument_group('plugin options')
for check in get_plugins('check', plugins):
add_addon(check)
for addon in all_addons:
addon.mangle_argparser(argparser)
@argparser.bind_final_check
def check_args(parser, namespace):
# XXX hack...
namespace.checks = sorted(unstable_unique(get_plugins('check', plugins)),
key=lambda x: x.__name__)
if any((namespace.list_keywords, namespace.list_checks,
namespace.list_reporters)):
# no need to check any other args
return
-k/--keywords option.
Available scopes: %s
""" % (', '.join(base.known_scopes)))
def add_addon(addon, addon_set):
if addon not in addon_set:
addon_set.add(addon)
for dep in addon.required_addons:
add_addon(dep, addon_set)
all_addons = set()
scan.plugin = scan.add_argument_group('plugin options')
for check in get_plugins('check', plugins):
add_addon(check, all_addons)
for addon in all_addons:
addon.mangle_argparser(scan)
# XXX hack...
_known_checks = tuple(
sorted(unstable_unique(get_plugins('check', plugins)),
key=lambda x: x.__name__))
_known_keywords = tuple(
sorted(unstable_unique(
chain.from_iterable(check.known_results for check in _known_checks)),
key=lambda x: x.__name__))
@scan.bind_final_check
for addon in options.addons:
# Ignore the return value, we just need to populate addons_map.
init_addon(addon)
if options.debug:
err.write('target repo: ', repr(options.target_repo))
err.write('source repo: ', repr(options.src_repo))
err.write('base dirs: ', repr(options.repo_bases))
for filterer in options.limiters:
err.write('limiter: ', repr(filterer))
debug = logging.debug
else:
debug = None
transforms = list(get_plugins('transform', plugins))
# XXX this is pretty horrible.
sinks = list(addon for addon in addons_map.itervalues()
if getattr(addon, 'feed_type', False))
reporter.start()
for filterer in options.limiters:
sources = [feeds.RestrictedRepoSource(options.target_repo, filterer)]
bad_sinks, pipes = base.plug(sinks, transforms, sources, debug)
if bad_sinks:
# We want to report the ones that would work if this was a
# full repo scan separately from the ones that are
# actually missing transforms.
bad_sinks = set(bad_sinks)
full_scope = feeds.RestrictedRepoSource(options.target_repo,
def check_values(self, values, args):
values, args = commandline.OptionParser.check_values(
self, values, args)
# XXX hack...
values.checks = sorted(lists.unstable_unique(
get_plugins('check', plugins)),
key=lambda x:x.__name__)
if values.list_checks or values.list_reporters:
if values.list_reporters == values.list_checks:
raise optparse.OptionValueError("--list-checks and "
"--list-reporters are mutually exclusive options- "
"one or the other.")
return values, ()
cwd = None
if values.suite is None:
# No suite explicitly specified. Use the repo to guess the suite.
if values.target_repo is None:
# Not specified either. Try to find a repo our cwd is in.
cwd = os.getcwd()
# The use of a dict here is a hack to deal with one
# repo having multiple names in the configuration.
candidates = {}
for name, suite in values.config.pcheck_suite.iteritems():
repo = suite.target_repo
if repo is None:
continue
repo_base = getattr(repo, 'base', None)
if repo_base is not None and cwd.startswith(repo_base):
candidates[repo] = name
if len(candidates) == 1:
values.guessed_suite = True
values.target_repo = tuple(candidates)[0]
if values.target_repo is not None:
# We have a repo, now find a suite matching it.
candidates = list(
suite for suite in values.config.pcheck_suite.itervalues()
if suite.target_repo is values.target_repo)
if len(candidates) == 1:
values.guessed_suite = True
values.suite = candidates[0]
if values.suite is None:
# If we have multiple candidates or no candidates we
# fall back to the default suite.
values.suite = values.config.get_default('pcheck_suite')
values.default_suite = values.suite is not None
if values.suite is not None:
# We have a suite. Lift defaults from it for values that
# were not set explicitly:
if values.checkset is None:
values.checkset = values.suite.checkset
if values.src_repo is None:
values.src_repo = values.suite.src_repo
# If we were called with no atoms we want to force
# cwd-based detection.
if values.target_repo is None:
if args:
values.target_repo = values.suite.target_repo
elif values.suite.target_repo is not None:
# No atoms were passed in, so we want to guess
# what to scan based on cwd below. That only makes
# sense if we are inside the target repo. We still
# want to pick the suite's target repo if we are
# inside it, in case there is more than one repo
# definition with a base that contains our dir.
if cwd is None:
cwd = os.getcwd()
repo_base = getattr(values.suite.target_repo, 'base', None)
if repo_base is not None and cwd.startswith(repo_base):
values.target_repo = values.suite.target_repo
if values.target_repo is None:
# We have no target repo (not explicitly passed, not from
# a suite, not from an earlier guess at the target_repo).
# Try to guess one from cwd:
if cwd is None:
cwd = os.getcwd()
candidates = {}
for name, repo in values.config.repo.iteritems():
repo_base = getattr(repo, 'base', None)
if repo_base is not None and cwd.startswith(repo_base):
candidates[repo] = name
if not candidates:
self.error(
'No target repo specified on commandline or suite and '
'current directory is not inside a known repo.')
elif len(candidates) > 1:
self.error(
'Found multiple matches when guessing repo based on '
'current directory (%s). Specify a repo on the '
'commandline or suite or remove some repos from your '
'configuration.' % (
', '.join(str(repo) for repo in candidates),))
values.target_repo = tuple(candidates)[0]
if values.reporter is None:
values.reporter = values.config.get_default(
'pcheck_reporter_factory')
if values.reporter is None:
values.reporter = get_plugin('reporter', plugins)
if values.reporter is None:
self.error('no config defined reporter found, nor any default '
'plugin based reporters')
else:
func = values.config.pcheck_reporter_factory.get(values.reporter)
if func is None:
func = list(base.Whitelist([values.reporter]).filter(
get_plugins('reporter', plugins)))
if not func:
self.error("no reporter matches %r\n"
"please see --list-reporter for a list of "
"valid reporters" % values.reporter)
elif len(func) > 1:
self.error("--reporter %r matched multiple reporters, "
"must match one. %r" %
(values.reporter,
tuple(sorted("%s.%s" %
(x.__module__, x.__name__)
for x in func))
)
)
func = func[0]
values.reporter = func
if values.src_repo is None:
values.src_repo = values.target_repo
values.search_repo = values.target_repo
else:
values.search_repo = multiplex.tree(values.target_repo,
values.src_repo)
# TODO improve this to deal with a multiplex repo.
for repo in set((values.src_repo, values.target_repo)):
if isinstance(repo, repository.UnconfiguredTree):
values.repo_bases.append(osutils.abspath(repo.base))
if args:
values.limiters = lists.stable_unique(map(
parserestrict.parse_match, args))
else:
repo_base = getattr(values.target_repo, 'base', None)
if not repo_base:
self.error(
'Either specify a target repo that is not multi-tree or '
'one or more extended atoms to scan '
'("*" for the entire repo).')
cwd = osutils.abspath(os.getcwd())
repo_base = osutils.abspath(repo_base)
if not cwd.startswith(repo_base):
self.error(
'Working dir (%s) is not inside target repo (%s). Fix '
'that or specify one or more extended atoms to scan.' % (
cwd, repo_base))
bits = list(p for p in cwd[len(repo_base):].split(os.sep) if p)
if not bits:
values.limiters = [packages.AlwaysTrue]
elif len(bits) == 1:
values.limiters = [packages.PackageRestriction(
'category', StrExactMatch(bits[0]))]
else:
values.limiters = [packages.AndRestriction(
packages.PackageRestriction(
'category', StrExactMatch(bits[0])),
packages.PackageRestriction(
'package', StrExactMatch(bits[1])))]
if values.checkset is None:
values.checkset = values.config.get_default('pcheck_checkset')
if values.checkset is not None:
values.checks = list(values.checkset.filter(values.checks))
if values.checks_to_run:
whitelist = base.Whitelist(values.checks_to_run)
values.checks = list(whitelist.filter(values.checks))
if values.checks_to_disable:
blacklist = base.Blacklist(values.checks_to_disable)
values.checks = list(blacklist.filter(values.checks))
if not values.checks:
self.error('No active checks')
values.addons = set()
def add_addon(addon):
if addon not in values.addons:
values.addons.add(addon)
for dep in addon.required_addons:
add_addon(dep)
for check in values.checks:
add_addon(check)
try:
for addon in values.addons:
addon.check_values(values)
except optparse.OptionValueError, e:
if values.debug:
raise
self.error(str(e))
def __init__(self, **kwargs):
commandline.OptionParser.__init__(
self, version='pkgcore-checks %s' % (__version__,),
description="pkgcore based ebuild QA checks",
usage="usage: %prog [options] [atom1...atom2]",
**kwargs)
# These are all set in check_values based on other options, so have
# no default set through add_option.
self.set_default('repo_bases', [])
self.set_default('guessed_target_repo', False)
self.set_default('guessed_suite', False)
self.set_default('default_suite', False)
group = self.add_option_group('Check selection')
group.add_option(
"-c", action="append", type="string", dest="checks_to_run",
help="limit checks to those matching this regex, or package/class "
"matching; may be specified multiple times")
group.set_conflict_handler("resolve")
group.add_option("-d",
"--disable", action="append", type="string",
dest="checks_to_disable", help="specific checks to disable: "
"may be specified multiple times")
group.set_conflict_handler("error")
group.add_option(
'--checkset', action='callback', type='string',
callback=commandline.config_callback,
callback_args=('pcheck_checkset', 'checkset'),
help='Pick a preconfigured set of checks to run.')
self.add_option(
'--repo', '-r', action='callback', type='string',
callback=repo_callback, dest='target_repo',
help='Set the target repo')
self.add_option(
'--suite', '-s', action='callback', type='string',
callback=commandline.config_callback,
callback_args=('pcheck_suite', 'suite'),
help='Specify the configuration suite to use')
self.add_option(
"--list-checks", action="store_true", default=False,
help="print what checks are available to run and exit")
self.add_option(
'--reporter', type='string', action='store', default=None,
help="Use a non-default reporter (defined in pkgcore's config).")
self.add_option(
'--list-reporters', action='store_true', default=False,
help="print known reporters")
overlay = self.add_option_group('Overlay')
overlay.add_option(
'--overlayed-repo', '-o', action='callback', type='string',
callback=repo_callback, dest='src_repo',
help="if the target repository is an overlay, specify the "
"repository name to pull profiles/license from")
all_addons = set()
def add_addon(addon):
if addon not in all_addons:
all_addons.add(addon)
for dep in addon.required_addons:
add_addon(dep)
for check in get_plugins('check', plugins):
add_addon(check)
for addon in all_addons:
addon.mangle_option_parser(self)
def _validate_args(parser, namespace):
namespace.enabled_checks = list(_known_checks)
namespace.enabled_keywords = list(_known_keywords)
cwd = abspath(os.getcwd())
if namespace.suite is None:
# No suite explicitly specified. Use the repo to guess the suite.
if namespace.target_repo is None:
# Not specified either. Try to find a repo our cwd is in.
# The use of a dict here is a hack to deal with one
# repo having multiple names in the configuration.
candidates = {}
for name, suite in namespace.config.pkgcheck_suite.items():
repo = suite.target_repo
if repo is None:
continue
repo_base = getattr(repo, 'location', None)
if repo_base is not None and cwd.startswith(repo_base):
candidates[repo] = name
if len(candidates) == 1:
namespace.guessed_suite = True
namespace.target_repo = tuple(candidates)[0]
if namespace.target_repo is not None:
# We have a repo, now find a suite matching it.
candidates = list(
suite for suite in namespace.config.pkgcheck_suite.values()
if suite.target_repo is namespace.target_repo)
if len(candidates) == 1:
namespace.guessed_suite = True
namespace.suite = candidates[0]
if namespace.suite is None:
# If we have multiple candidates or no candidates we
# fall back to the default suite.
namespace.suite = namespace.config.get_default('pkgcheck_suite')
namespace.default_suite = namespace.suite is not None
if namespace.suite is not None:
# We have a suite. Lift defaults from it for values that
# were not set explicitly:
if namespace.checkset is None:
namespace.checkset = namespace.suite.checkset
# If we were called with no atoms we want to force
# cwd-based detection.
if namespace.target_repo is None:
if namespace.targets:
namespace.target_repo = namespace.suite.target_repo
elif namespace.suite.target_repo is not None:
# No atoms were passed in, so we want to guess
# what to scan based on cwd below. That only makes
# sense if we are inside the target repo. We still
# want to pick the suite's target repo if we are
# inside it, in case there is more than one repo
# definition with a base that contains our dir.
repo_base = getattr(namespace.suite.target_repo, 'location',
None)
if repo_base is not None and cwd.startswith(repo_base):
namespace.target_repo = namespace.suite.target_repo
if namespace.target_repo is None:
# We have no target repo (not explicitly passed, not from a suite, not
# from an earlier guess at the target_repo) so try to guess one.
target_repo = None
target_dir = cwd
# pull a target directory from target args if they're path-based
if namespace.targets and os.path.exists(namespace.targets[0]):
target = namespace.targets[0]
if os.path.isfile(target):
target = os.path.dirname(target)
target_dir = target
# determine target repo from the target directory
for repo in namespace.domain.ebuild_repos_raw:
if target_dir in repo:
target_repo = repo
break
# fallback to the default repo
if target_repo is None:
target_repo = namespace.config.get_default('repo')
namespace.target_repo = target_repo
# use filtered repo if filtering is enabled
if namespace.filtered:
namespace.target_repo = namespace.domain.ebuild_repos[str(
namespace.target_repo)]
if namespace.reporter is None:
namespace.reporter = namespace.config.get_default(
'pkgcheck_reporter_factory')
if namespace.reporter is None:
namespace.reporter = get_plugin('reporter', plugins)
if namespace.reporter is None:
parser.error('no config defined reporter found, nor any default '
'plugin based reporters')
else:
func = namespace.config.pkgcheck_reporter_factory.get(
namespace.reporter)
if func is None:
func = list(
base.Whitelist([namespace.reporter
]).filter(get_plugins('reporter', plugins)))
if not func:
available = ', '.join(
sorted(x.__name__
for x in get_plugins('reporter', plugins)))
parser.error(f"no reporter matches {namespace.reporter!r} "
f"(available: {available})")
elif len(func) > 1:
reporters = tuple(
sorted(f"{x.__module__}.{x.__name__}" for x in func))
parser.error(
f"reporter {namespace.reporter!r} matched multiple reporters, "
f"must match one. {reporters!r}")
func = func[0]
namespace.reporter = func
# search_repo is a multiplex of target_repo and its masters, make sure
# they're configured properly in metadata/layout.conf. This is used for
# things like visibility checks (it is passed to the checkers in "start").
namespace.search_repo = multiplex.tree(*namespace.target_repo.trees)
namespace.repo_bases = [
abspath(repo.location)
for repo in reversed(namespace.target_repo.trees)
]
namespace.default_target = None
if namespace.targets:
repo = namespace.target_repo
# read targets from stdin in a non-blocking manner
if len(namespace.targets) == 1 and namespace.targets[0] == '-':
def stdin():
while True:
line = sys.stdin.readline()
if not line:
break
yield line.rstrip()
namespace.targets = stdin()
def limiters():
for target in namespace.targets:
try:
yield parserestrict.parse_match(target)
except parserestrict.ParseError as e:
if os.path.exists(target):
try:
yield repo.path_restrict(target)
except ValueError as e:
parser.error(e)
else:
parser.error(e)
namespace.limiters = limiters()
else:
repo_base = getattr(namespace.target_repo, 'location', None)
if not repo_base:
parser.error(
'Either specify a target repo that is not multi-tree or '
'one or more extended atoms to scan '
'("*" for the entire repo).')
if cwd not in namespace.target_repo:
namespace.limiters = [packages.AlwaysTrue]
else:
namespace.limiters = [
packages.AndRestriction(
*namespace.target_repo.path_restrict(cwd))
]
namespace.default_target = cwd
if namespace.checkset is None:
namespace.checkset = namespace.config.get_default('pkgcheck_checkset')
if namespace.checkset is not None:
namespace.enabled_checks = list(
namespace.checkset.filter(namespace.enabled_checks))
if namespace.selected_scopes is not None:
disabled_scopes, enabled_scopes = namespace.selected_scopes
# validate selected scopes
selected_scopes = set(disabled_scopes + enabled_scopes)
unknown_scopes = selected_scopes - set(base.known_scopes.keys())
if unknown_scopes:
parser.error('unknown scope%s: %s (available scopes: %s)' %
(_pl(unknown_scopes), ', '.join(unknown_scopes),
', '.join(base.known_scopes.keys())))
# convert scopes to keyword lists
disabled_keywords = [
k.__name__ for s in disabled_scopes for k in _known_keywords
if k.threshold == base.known_scopes[s]
]
enabled_keywords = [
k.__name__ for s in enabled_scopes for k in _known_keywords
if k.threshold == base.known_scopes[s]
]
# filter outputted keywords
namespace.enabled_keywords = base.filter_update(
namespace.enabled_keywords, enabled_keywords, disabled_keywords)
if namespace.selected_keywords is not None:
disabled_keywords, enabled_keywords = namespace.selected_keywords
errors = (x.__name__ for x in _known_keywords
if issubclass(x, base.Error))
warnings = (x.__name__ for x in _known_keywords
if issubclass(x, base.Warning))
alias_map = {'errors': errors, 'warnings': warnings}
replace_aliases = lambda x: alias_map.get(x, [x])
# expand keyword aliases to keyword lists
disabled_keywords = list(
chain.from_iterable(map(replace_aliases, disabled_keywords)))
enabled_keywords = list(
chain.from_iterable(map(replace_aliases, enabled_keywords)))
# validate selected keywords
selected_keywords = set(disabled_keywords + enabled_keywords)
available_keywords = set(x.__name__ for x in _known_keywords)
unknown_keywords = selected_keywords - available_keywords
if unknown_keywords:
parser.error(
"unknown keyword%s: %s (use 'pkgcheck show --keywords' to show valid keywords)"
% (_pl(unknown_keywords), ', '.join(unknown_keywords)))
# filter outputted keywords
namespace.enabled_keywords = base.filter_update(
namespace.enabled_keywords, enabled_keywords, disabled_keywords)
namespace.filtered_keywords = set(namespace.enabled_keywords)
if namespace.filtered_keywords == set(_known_keywords):
namespace.filtered_keywords = None
disabled_checks, enabled_checks = ((), ())
if namespace.selected_checks is not None:
disabled_checks, enabled_checks = namespace.selected_checks
# validate selected checks
selected_checks = set(disabled_checks + enabled_checks)
available_checks = set(x.__name__ for x in _known_checks)
unknown_checks = selected_checks - available_checks
if unknown_checks:
parser.error(
"unknown check%s: %r (use 'pkgcheck show --checks' to show valid checks)"
% (_pl(unknown_checks), ', '.join(unknown_checks)))
elif namespace.filtered_keywords is not None:
# enable checks based on enabled keyword -> check mapping
enabled_checks = []
for check in _known_checks:
if namespace.filtered_keywords.intersection(check.known_results):
enabled_checks.append(check.__name__)
# filter checks to run
if enabled_checks:
whitelist = base.Whitelist(enabled_checks)
namespace.enabled_checks = list(
whitelist.filter(namespace.enabled_checks))
if disabled_checks:
blacklist = base.Blacklist(disabled_checks)
namespace.enabled_checks = list(
blacklist.filter(namespace.enabled_checks))
if not namespace.enabled_checks:
parser.error('no active checks')
namespace.addons = set()
for check in namespace.enabled_checks:
add_addon(check, namespace.addons)
try:
for addon in namespace.addons:
addon.check_args(parser, namespace)
except argparse.ArgumentError as e:
if namespace.debug:
raise
parser.error(str(e))
def main(options, out, err):
"""Do stuff."""
if options.list_keywords:
display_keywords(out, options)
return 0
if options.list_checks:
display_checks(out, options)
return 0
if options.list_reporters:
display_reporters(out, options,
options.config.pkgcheck_reporter_factory.values(),
list(get_plugins('reporter', plugins)))
return 0
if not options.repo_bases:
err.write(
'Warning: could not determine repo base for profiles, some checks will not work.'
)
err.write()
if options.guessed_suite:
if options.default_suite:
err.write('Tried to guess a suite to use but got multiple matches')
err.write('and fell back to the default.')
else:
err.write('using suite guessed from working directory')
if options.guessed_target_repo:
err.write('using repository guessed from working directory')
try:
reporter = options.reporter(out)
except errors.ReporterInitError as e:
err.write(err.fg('red'), err.bold, '!!! ', err.reset,
'Error initializing reporter: ', e)
return 1
addons_map = {}
def init_addon(klass):
res = addons_map.get(klass)
if res is not None:
return res
deps = list(init_addon(dep) for dep in klass.required_addons)
try:
res = addons_map[klass] = klass(options, *deps)
except KeyboardInterrupt:
raise
except Exception:
err.write('instantiating %s' % (klass, ))
raise
return res
for addon in options.addons:
# Ignore the return value, we just need to populate addons_map.
init_addon(addon)
if options.debug:
err.write('target repo: ', repr(options.target_repo))
err.write('base dirs: ', repr(options.repo_bases))
for filterer in options.limiters:
err.write('limiter: ', repr(filterer))
debug = logging.debug
else:
debug = None
transforms = list(get_plugins('transform', plugins))
# XXX this is pretty horrible.
sinks = list(addon for addon in addons_map.itervalues()
if getattr(addon, 'feed_type', False))
reporter.start()
for filterer in options.limiters:
sources = [feeds.RestrictedRepoSource(options.target_repo, filterer)]
bad_sinks, pipes = base.plug(sinks, transforms, sources, debug)
if bad_sinks:
# We want to report the ones that would work if this was a
# full repo scan separately from the ones that are
# actually missing transforms.
bad_sinks = set(bad_sinks)
full_scope = feeds.RestrictedRepoSource(options.target_repo,
packages.AlwaysTrue)
really_bad, ignored = base.plug(sinks, transforms, [full_scope])
really_bad = set(really_bad)
assert bad_sinks >= really_bad, \
'%r unreachable with no limiters but reachable with?' % (
really_bad - bad_sinks,)
for sink in really_bad:
err.error(
'sink %s could not be connected (missing transforms?)' %
(sink, ))
out_of_scope = bad_sinks - really_bad
if options.verbose and out_of_scope:
err.warn('skipping repo checks (not a full repo scan)')
if not pipes:
out.write(out.fg('red'), ' * ', out.reset, 'No checks!')
else:
if options.debug:
err.write('Running %i tests' % (len(sinks) - len(bad_sinks), ))
for source, pipe in pipes:
pipe.start()
reporter.start_check(list(base.collect_checks_classes(pipe)),
filterer)
for thing in source.feed():
pipe.feed(thing, reporter)
pipe.finish(reporter)
reporter.end_check()
reporter.finish()
# flush stdout first; if they're directing it all to a file, this makes
# results not get the final message shoved in midway
out.stream.flush()
return 0
from textwrap import dedent
from pkgcore.plugin import get_plugins
from snakeoil.sequences import unstable_unique
from pkgcheck import plugins
def _rst_header(char, text, newline=True):
if newline:
print('\n', end='')
print(text)
print(char * len(text))
checks = sorted(unstable_unique(get_plugins('check', plugins)),
key=lambda x: x.__name__)
d = defaultdict(set)
for check in checks:
d[check.scope].add(check)
_rst_header('=', 'Checks', newline=False)
scopes = ('version', 'package', 'category', 'repository')
for scope in reversed(sorted(d)):
_rst_header('-', scopes[scope].capitalize() + ' scope')
checks = sorted(d[scope], key=lambda x: x.__name__)
for check in checks:
if check.__doc__ is not None:
def check_args(parser, namespace):
# XXX hack...
namespace.checks = sorted(unstable_unique(get_plugins('check', plugins)),
key=lambda x: x.__name__)
if any((namespace.list_keywords, namespace.list_checks,
namespace.list_reporters)):
# no need to check any other args
return
cwd = abspath(os.getcwd())
if namespace.suite is None:
# No suite explicitly specified. Use the repo to guess the suite.
if namespace.target_repo is None:
# Not specified either. Try to find a repo our cwd is in.
# The use of a dict here is a hack to deal with one
# repo having multiple names in the configuration.
candidates = {}
for name, suite in namespace.config.pkgcheck_suite.iteritems():
repo = suite.target_repo
if repo is None:
continue
repo_base = getattr(repo, 'location', None)
if repo_base is not None and cwd.startswith(repo_base):
candidates[repo] = name
if len(candidates) == 1:
namespace.guessed_suite = True
namespace.target_repo = tuple(candidates)[0]
if namespace.target_repo is not None:
# We have a repo, now find a suite matching it.
candidates = list(
suite
for suite in namespace.config.pkgcheck_suite.itervalues()
if suite.target_repo is namespace.target_repo)
if len(candidates) == 1:
namespace.guessed_suite = True
namespace.suite = candidates[0]
if namespace.suite is None:
# If we have multiple candidates or no candidates we
# fall back to the default suite.
namespace.suite = namespace.config.get_default('pkgcheck_suite')
namespace.default_suite = namespace.suite is not None
if namespace.suite is not None:
# We have a suite. Lift defaults from it for values that
# were not set explicitly:
if namespace.checkset is None:
namespace.checkset = namespace.suite.checkset
# If we were called with no atoms we want to force
# cwd-based detection.
if namespace.target_repo is None:
if namespace.targets:
namespace.target_repo = namespace.suite.target_repo
elif namespace.suite.target_repo is not None:
# No atoms were passed in, so we want to guess
# what to scan based on cwd below. That only makes
# sense if we are inside the target repo. We still
# want to pick the suite's target repo if we are
# inside it, in case there is more than one repo
# definition with a base that contains our dir.
repo_base = getattr(namespace.suite.target_repo, 'location',
None)
if repo_base is not None and cwd.startswith(repo_base):
namespace.target_repo = namespace.suite.target_repo
if namespace.target_repo is None:
# We have no target repo (not explicitly passed, not from a suite, not
# from an earlier guess at the target_repo) so try to guess one.
if len(namespace.targets) == 1 and os.path.exists(
namespace.targets[0]):
target_dir = namespace.targets[0]
else:
target_dir = cwd
target_repo = None
for name, repo in namespace.config.repo.iteritems():
repo_base = getattr(repo, 'location', None)
if repo_base is not None and target_dir in repo:
target_repo = repo
if target_repo is None:
parser.error('no target repo specified and '
'current directory is not inside a known repo')
namespace.target_repo = target_repo
if namespace.reporter is None:
namespace.reporter = namespace.config.get_default(
'pkgcheck_reporter_factory')
if namespace.reporter is None:
namespace.reporter = get_plugin('reporter', plugins)
if namespace.reporter is None:
parser.error('no config defined reporter found, nor any default '
'plugin based reporters')
else:
func = namespace.config.pkgcheck_reporter_factory.get(
namespace.reporter)
if func is None:
func = list(
base.Whitelist([namespace.reporter
]).filter(get_plugins('reporter', plugins)))
if not func:
parser.error(
"no reporter matches %r (available: %s)" %
(namespace.reporter, ', '.join(
sorted(x.__name__
for x in get_plugins('reporter', plugins)))))
elif len(func) > 1:
parser.error("--reporter %r matched multiple reporters, "
"must match one. %r" %
(namespace.reporter,
tuple(
sorted("%s.%s" % (x.__module__, x.__name__)
for x in func))))
func = func[0]
namespace.reporter = func
# search_repo is a multiplex of target_repo and its masters, make sure
# they're configured properly in metadata/layout.conf. This is used for
# things like visibility checks (it is passed to the checkers in "start").
namespace.search_repo = multiplex.tree(*namespace.target_repo.trees)
namespace.repo_bases = [
abspath(repo.location)
for repo in reversed(namespace.target_repo.trees)
]
if namespace.targets:
limiters = []
repo = namespace.target_repo
# read targets from stdin
if len(namespace.targets) == 1 and namespace.targets[0] == '-':
namespace.targets = [
x.strip() for x in sys.stdin.readlines() if x.strip() != ''
]
# reassign stdin to allow interactivity (currently only works for unix)
sys.stdin = open('/dev/tty')
for target in namespace.targets:
try:
limiters.append(parserestrict.parse_match(target))
except parserestrict.ParseError as e:
if os.path.exists(target):
try:
limiters.append(repo.path_restrict(target))
except ValueError as e:
parser.error(e)
else:
parser.error(e)
namespace.limiters = limiters
else:
repo_base = getattr(namespace.target_repo, 'location', None)
if not repo_base:
parser.error(
'Either specify a target repo that is not multi-tree or '
'one or more extended atoms to scan '
'("*" for the entire repo).')
if cwd not in namespace.target_repo:
namespace.limiters = [packages.AlwaysTrue]
else:
namespace.limiters = [
packages.AndRestriction(
*namespace.target_repo.path_restrict(cwd))
]
if namespace.checkset is None:
namespace.checkset = namespace.config.get_default('pkgcheck_checkset')
if namespace.checkset is not None:
namespace.checks = list(namespace.checkset.filter(namespace.checks))
disabled_checks, enabled_checks = ((), ())
if namespace.selected_checks is not None:
disabled_checks, enabled_checks = namespace.selected_checks
if enabled_checks:
whitelist = base.Whitelist(enabled_checks)
namespace.checks = list(whitelist.filter(namespace.checks))
if disabled_checks:
blacklist = base.Blacklist(disabled_checks)
namespace.checks = list(blacklist.filter(namespace.checks))
if not namespace.checks:
parser.error('no active checks')
namespace.addons = set()
def add_addon(addon):
if addon not in namespace.addons:
namespace.addons.add(addon)
for dep in addon.required_addons:
add_addon(dep)
for check in namespace.checks:
add_addon(check)
try:
for addon in namespace.addons:
addon.check_args(parser, namespace)
except argparse.ArgumentError as e:
if namespace.debug:
raise
parser.error(str(e))
from textwrap import dedent
from pkgcore.plugin import get_plugins
from snakeoil.sequences import unstable_unique
from pkgcheck import plugins
def _rst_header(char, text, newline=True):
if newline:
print('\n', end='')
print(text)
print(char * len(text))
reporters = sorted(unstable_unique(get_plugins('reporter', plugins)),
key=lambda x: x.__name__)
_rst_header('=', 'Reporters', newline=False)
for reporter in reporters:
if reporter.__doc__ is not None:
try:
summary, explanation = reporter.__doc__.split('\n', 1)
except ValueError:
summary = reporter.__doc__
explanation = None
else:
summary = None
print('\n{}'.format(reporter.__name__))
def AutodetectSyncer(basedir, **kwargs):
for plug in plugin.get_plugins('syncer'):
ret = plug.is_usable_on_filepath(basedir)
if ret is not None:
return plug(basedir, *ret, **kwargs)
return DisabledSyncer(basedir, **kwargs)
