def test_ReportDependencies(self):
m = checkpkg_lib.CheckpkgManager2("testname", [], "5.9", "sparc",
"unstable")
checkpkg_interface_mock = self.mox.CreateMock(
checkpkg_lib.IndividualCheckInterface)
needed_files = [
("CSWfoo", "/opt/csw/bin/needed_file", "reason1"),
]
needed_pkgs = []
messenger_stub = stubs.MessengerStub()
declared_deps_by_pkgname = {
"CSWfoo": frozenset([
"CSWbar-1",
"CSWbar-2",
]),
}
checkpkg_interface_mock.GetPkgByPath(
'/opt/csw/bin/needed_file').AndReturn(["CSWfoo-one", "CSWfoo-two"])
checkpkg_interface_mock.ReportErrorForPkgname(
'CSWfoo', 'missing-dependency', 'CSWfoo-one or CSWfoo-two')
checkpkg_interface_mock.ReportErrorForPkgname('CSWfoo',
'surplus-dependency',
'CSWbar-2')
checkpkg_interface_mock.ReportErrorForPkgname('CSWfoo',
'surplus-dependency',
'CSWbar-1')
self.mox.ReplayAll()
m._ReportDependencies(checkpkg_interface_mock, needed_files,
needed_pkgs, messenger_stub,
declared_deps_by_pkgname)
def test_RemovePkgsFromMissing(self):
m = checkpkg_lib.CheckpkgManager2("testname", [], "5.9", "sparc",
"unstable")
missing_dep_groups = [['CSWfoo-one', 'CSWfoo']]
expected = set([
frozenset(['CSWfoo', 'CSWfoo-one']),
])
result = m._RemovePkgsFromMissing("CSWbaz", missing_dep_groups)
self.assertEqual(expected, result)
def testSurplusDeps(self):
m = checkpkg_lib.CheckpkgManager2("testname", [], "5.9", "sparc",
"unstable")
potential_req_pkgs = set([u"CSWbar"])
declared_deps = set([u"CSWbar", u"CSWsurplus"])
expected = set(["CSWsurplus"])
self.assertEquals(
expected,
m._GetSurplusDeps("CSWfoo", potential_req_pkgs, declared_deps))
def testSingleTag(self):
m = checkpkg_lib.CheckpkgManager2("testname", [], "5.9", "sparc",
"unstable")
tags = {
"CSWfoo": [
tag.CheckpkgTag("CSWfoo", "foo-tag", "foo-info"),
],
}
screen_report, tags_report = m.FormatReports(tags, [], [])
expected = u'# Tags reported by testname module\nCSWfoo: foo-tag foo-info\n'
self.assertEqual(expected, unicode(tags_report))
def testMissingDepsFromReasonGroupsSelf(self):
m = checkpkg_lib.CheckpkgManager2("testname", [], "5.9", "sparc",
"unstable")
reason_groups = [
[(u"CSWfoo", "reason 1"), (u"CSWfoo2", "reason 1")],
]
declared_deps = set([])
expected = []
result = m._MissingDepsFromReasonGroups("CSWfoo", reason_groups,
declared_deps)
self.assertEqual(result, expected)
def testReportMissingDependenciesTwo(self):
m = checkpkg_lib.CheckpkgManager2("testname", [], "5.9", "sparc",
"unstable")
error_mgr_mock = self.mox.CreateMock(
checkpkg_lib.IndividualCheckInterface)
declared_deps = frozenset([])
req_pkgs_reasons = [
[
(u"CSWfoo-1", "reason 1"),
(u"CSWfoo-2", "reason 1"),
],
]
error_mgr_mock.ReportErrorForPkgname('CSWexamined',
'missing-dependency',
u'CSWfoo-1 or CSWfoo-2')
self.mox.ReplayAll()
m._ReportMissingDependencies(error_mgr_mock, "CSWexamined",
declared_deps, req_pkgs_reasons)
def test_ReportDependenciesDirProvidedBySelf(self):
m = checkpkg_lib.CheckpkgManager2("testname", [], "5.9", "sparc",
"unstable")
checkpkg_interface_mock = self.mox.CreateMock(
checkpkg_lib.IndividualCheckInterface)
needed_files = [
("CSWfoo", "/opt/csw/share/man/man1m", "reason1"),
]
needed_pkgs = []
messenger_stub = stubs.MessengerStub()
declared_deps_by_pkgname = {"CSWfoo": frozenset()}
checkpkg_interface_mock.GetPkgByPath(
'/opt/csw/share/man/man1m').AndReturn(
["CSWfoo", "CSWfoo-one", "CSWfoo-two"])
# Should not report any dependencies; the /opt/csw/share/man/man1m path is
# provided by the package itself.
self.mox.ReplayAll()
m._ReportDependencies(checkpkg_interface_mock, needed_files,
needed_pkgs, messenger_stub,
declared_deps_by_pkgname)
def DisabledtestReportMissingDependenciesIntegration(self):
m = checkpkg_lib.CheckpkgManager2("testname", [], "5.9", "sparc",
"unstable")
catalog_mock = self.mox.CreateMock(checkpkg_lib.Catalog)
checkpkg_interface = checkpkg_lib.IndividualCheckInterface(
"CSWfoo", "AlienOS5.2", "sparkle", "calcified", catalog_mock)
declared_deps_by_pkgname = {
"CSWfoo": frozenset(),
}
declared_deps = frozenset([])
pkgs_providing_path = ["CSWproviding-%02d" % x for x in range(20)]
catalog_mock.GetPkgByPath('/opt/csw/sbin', 'AlienOS5.2', 'sparkle',
'calcified').AndReturn(pkgs_providing_path)
self.mox.ReplayAll()
checkpkg_interface.NeedFile("/opt/csw/sbin", "reason 1")
needed_files = checkpkg_interface.needed_files
needed_pkgs = checkpkg_interface.needed_pkgs
messenger_stub = stubs.MessengerStub()
m._ReportDependencies(checkpkg_interface, needed_files, needed_pkgs,
messenger_stub, declared_deps_by_pkgname)
self.assertEqual(1, len(checkpkg_interface.errors))
self.assertEqual(" or ".join(sorted(pkgs_providing_path)),
checkpkg_interface.errors[0].tag_info)
def testGetAllTags(self):
# Does not run any checks, because they are unregistered. However,
# needfile and needpkg mechanisms are active.
#
# Disabling this check for now, because there are issues with mocking out
# some of the objects.
# TODO(maciej): Enable this check again.
return
self.mox.StubOutWithMock(checkpkg_lib,
'IndividualCheckInterface',
use_mock_anything=True)
self.mox.StubOutWithMock(checkpkg_lib,
'SetCheckInterface',
use_mock_anything=True)
# checkpkg_interface_mock = self.mox.CreateMock(
# checkpkg_lib.IndividualCheckInterface)
# Throws:
# UnknownMethodCallError: Method called is not a member of the
# object: GetPkgByPath
checkpkg_interface_mock = self.mox.CreateMockAnything()
# checkpkg_interface_mock = self.mox.CreateMock(
# checkpkg_lib.IndividualCheckInterface)
set_interface_mock = self.mox.CreateMockAnything()
# checkpkg_interface_mock.GetPkgByPath("/opt/csw/bin/foo").AndReturn(
# ["CSWbar", "CSWbaz"])
set_interface_mock.errors = []
set_interface_mock.needed_files = []
set_interface_mock.needed_pkgs = []
checkpkg_interface_mock.errors = []
checkpkg_interface_mock.needed_files = [
checkpkg_lib.NeededFile("CSWneon", "/opt/csw/bin/foo", "Because!"),
]
checkpkg_interface_mock.needed_pkgs = []
self.mox.StubOutWithMock(checkpkg_lib,
'Catalog',
use_mock_anything=True)
checkpkg_lib.IndividualCheckInterface(
'CSWneon', '5.9', 'sparc', 'unstable',
catalog_mock).AndReturn(checkpkg_interface_mock)
checkpkg_lib.SetCheckInterface(
'CSWneon', '5.9', 'sparc', 'unstable',
catalog_mock).AndReturn(set_interface_mock)
stat_obj = self.mox.CreateMockAnything()
data_obj = self.mox.CreateMockAnything()
stat_obj.data_obj = data_obj
pkg_stats = copy.deepcopy(neon_stats.pkgstats)
# Resetting the dependencies so that it doesn't report surplus deps.
pkg_stats["depends"] = []
data_obj.pickle = cPickle.dumps(pkg_stats)
checkpkg_interface_mock.ReportErrorForPkgname('CSWneon',
'missing-dependency',
'CSWbar or CSWbaz')
catalog_mock.GetPkgByPath('/opt/csw/bin/foo', '5.9', 'sparc',
'unstable').AndReturn(["CSWbar", "CSWbaz"])
self.mox.ReplayAll()
m = checkpkg_lib.CheckpkgManager2("testname", [], "5.9", "sparc",
"unstable")
# m._AutoregisterChecks()
errors, messages, gar_lines = m.GetAllTags([stat_obj])
self.mox.VerifyAll()
# self.assertEquals(
# {'CSWneon': [tag.CheckpkgTag('CSWneon', 'missing-dependency', 'CSWbar or CSWbaz')]},
# errors)
expected_messages = [
u'Dependency issues of CSWneon:',
u'CSWbar is needed by CSWneon, because:',
u' - Because!',
u'RUNTIME_DEP_PKGS_CSWneon += CSWbar',
u'CSWbaz is needed by CSWneon, because:',
u' - Because!',
u'RUNTIME_DEP_PKGS_CSWneon += CSWbaz',
]
self.assertEquals(expected_messages, messages)
expected_gar_lines = [
'# One of the following:', ' RUNTIME_DEP_PKGS_CSWneon += CSWbar',
' RUNTIME_DEP_PKGS_CSWneon += CSWbaz',
'# (end of the list of alternative dependencies)'
]
self.assertEquals(expected_gar_lines, gar_lines)
def main():
parser = optparse.OptionParser(USAGE)
parser.add_option("-d",
"--debug",
dest="debug",
action="store_true",
default=False,
help="Switch on debugging messages")
parser.add_option("-q",
"--quiet",
dest="quiet",
action="store_true",
default=False,
help="Display less messages")
parser.add_option(
"--catalog-release",
dest="catrel",
default="current",
help="A catalog release: current, unstable, testing, stable.")
parser.add_option(
"-r",
"--os-releases",
dest="osrel_commas",
help=("Comma separated list of ['SunOS5.9', 'SunOS5.10'], "
"e.g. 'SunOS5.9,SunOS5.10'."))
parser.add_option("-a",
"--catalog-architecture",
dest="arch",
help="Architecture: i386, sparc.")
parser.add_option("--profile",
dest="profile",
default=False,
action="store_true",
help="Enable profiling (a developer option).")
options, args = parser.parse_args()
assert len(args), "The list of files or md5 sums must be not empty."
logging_level = logging.INFO
if options.quiet:
logging_level = logging.WARNING
elif options.debug:
# If both flags are set, debug wins.
logging_level = logging.DEBUG
fmt = '%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s'
logging.basicConfig(format=fmt, level=logging_level)
logging.debug("Starting.")
configuration.SetUpSqlobjectConnection()
err_msg_list = []
if not options.osrel_commas:
err_msg_list.append("Please specify --os-releases.")
if not options.arch:
err_msg_list.append("Please specify --catalog-architecture.")
if options.arch not in cc.PHYSICAL_ARCHITECTURES:
err_msg_list.append(
"Valid --catalog-architecture values are: %s, you passed: %r" %
(cc.PHYSICAL_ARCHITECTURES, options.arch))
if err_msg_list:
raise UsageError(" ".join(err_msg_list))
md5_sums_from_files = []
collector = package_stats.StatsCollector(logger=logging,
debug=options.debug)
# We need to separate files and md5 sums.
md5_sums, file_list = [], []
for arg in args:
if struct_util.IsMd5(arg):
md5_sums.append(arg)
else:
file_list.append(arg)
config = configuration.GetConfig()
username, password = rest.GetUsernameAndPassword()
rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get('rest', 'releases'),
username=username,
password=password)
if file_list:
def MakeEntry(file_name):
file_hash = hashlib.md5()
with open(file_name, "r") as fd:
chunk_size = 2 * 1024 * 1024
data = fd.read(chunk_size)
while data:
file_hash.update(data)
data = fd.read(chunk_size)
md5_sum = file_hash.hexdigest()
del file_hash
_, file_basename = os.path.split(file_name)
return {
'pkg_path': file_name,
'md5sum': md5_sum,
'file_basename': file_basename,
}
entries = [MakeEntry(x) for x in file_list]
md5_sums_from_files = collector.CollectStatsFromCatalogEntries(
entries, False)
for md5_sum in md5_sums_from_files:
if not rest_client.IsRegisteredLevelOne(md5_sum):
rest_client.RegisterLevelOne(md5_sum)
# We need the md5 sums of these files
md5_sums.extend(md5_sums_from_files)
assert md5_sums, "The list of md5 sums must not be empty."
logging.debug("md5_sums: %s", md5_sums)
osrel_list = options.osrel_commas.split(",")
logging.debug("Reading packages data from the database.")
# This part might need improvements in order to handle a whole
# catalog. On the other hand, if we already have the whole catalog in
# the database, we can do it altogether differently.
# Transforming the result to a list in order to force object
# retrieval.
sqo_pkgs = list(
models.Srv4FileStats.select(
sqlobject.IN(models.Srv4FileStats.q.md5_sum, md5_sums)))
tags_for_all_osrels = []
try:
sqo_catrel = models.CatalogRelease.selectBy(
name=options.catrel).getOne()
except sqlobject.main.SQLObjectNotFound as e:
logging.fatal("Fetching from the db has failed: catrel=%s",
repr(str(options.catrel)))
logging.fatal("Available catalog releases:")
sqo_catrels = models.CatalogRelease.select()
for sqo_catrel in sqo_catrels:
logging.fatal(" - %s", sqo_catrel.name)
raise
sqo_arch = models.Architecture.selectBy(name=options.arch).getOne()
for osrel in osrel_list:
sqo_osrel = models.OsRelease.selectBy(short_name=osrel).getOne()
VerifyContents(sqo_osrel, sqo_arch)
check_manager = checkpkg_lib.CheckpkgManager2(
CHECKPKG_MODULE_NAME,
sqo_pkgs,
osrel,
options.arch,
options.catrel,
debug=options.debug,
show_progress=(os.isatty(1) and not options.quiet))
# Running the checks, reporting and exiting.
exit_code, screen_report, tags_report = check_manager.Run()
screen_report = unicode(screen_report)
if not options.quiet and screen_report:
# TODO: Write this to screen only after overrides are applied.
sys.stdout.write(screen_report)
else:
logging.debug("No screen report.")
overrides_list = [list(pkg.GetOverridesResult()) for pkg in sqo_pkgs]
override_list = reduce(operator.add, overrides_list)
args = (sqo_osrel, sqo_arch, sqo_catrel)
tag_lists = [list(pkg.GetErrorTagsResult(*args)) for pkg in sqo_pkgs]
error_tags = reduce(operator.add, tag_lists)
(tags_after_overrides, unapplied_overrides) = overrides.ApplyOverrides(
error_tags, override_list)
tags_for_all_osrels.extend(tags_after_overrides)
if not options.quiet:
if tags_after_overrides:
print(textwrap.fill(BEFORE_OVERRIDES, 80))
for checkpkg_tag in tags_after_overrides:
print checkpkg_tag.ToGarSyntax()
print
for paragraph in AFTER_OVERRIDES:
print(textwrap.fill(paragraph, 80))
print
elif error_tags:
msg = (
'Fair enough, there were %d error tags, '
'but they were all overridden. '
"Just make sure you didn't override anything silly, like "
'sparc binaries in a i386 package.' % len(error_tags))
print
else:
print('Jolly good! All checks passed, no error tags reported.')
if unapplied_overrides:
print textwrap.fill(UNAPPLIED_OVERRIDES, 80)
for override in unapplied_overrides:
print u"* Unused %s" % override
exit_code = bool(tags_for_all_osrels)
sys.exit(exit_code)