def test_ParsePkgContents(self):
spi = system_pkgmap.Indexer()
srv4_stream = (PKGMAP_LINE_1, PKGMAP_LINE_2, PKGMAP_LINE_3,
PKGMAP_LINE_4)
ips_stream = (PKGCONTENT_LINE_1, PKGCONTENT_LINE_2, PKGCONTENT_LINE_3,
PKGCONTENT_LINE_4)
self.assertEqual(
4,
len(
spi._ParsePkgContents(srv4_stream, spi._ParseSrv4PkgmapLine,
False)))
self.assertEqual(
4,
len(
spi._ParsePkgContents(ips_stream, spi._ParseIpsPkgContentsLine,
False)))
def test_ParseIpsPkgContentsLineHardlink(self):
spi = system_pkgmap.Indexer()
line = PKGCONTENT_LINE_3
expected = representations.PkgmapEntry(
line=PKGCONTENT_LINE_3,
class_=None,
mode=None,
owner=None,
group=None,
type_='l',
major=None,
minor=None,
size=None,
path='/etc/svc/profile/platform_SUNW,UltraSPARC-IIe-NetraCT-40.xml',
target='./platform_SUNW,UltraSPARC-IIi-Netract.xml',
cksum=None,
modtime=None,
pkgnames=['SUNWsystem-core-os'],
)
self.assertEqual(expected, spi._ParseIpsPkgContentsLine(line))
def test_ParseIpsPkgContentsLineFile(self):
spi = system_pkgmap.Indexer()
line = PKGCONTENT_LINE_4
expected = representations.PkgmapEntry(
line=PKGCONTENT_LINE_4,
class_=None,
mode='0755',
owner='root',
group='bin',
type_='f',
major=None,
minor=None,
size=None,
path='/lib/libc.so.1',
target=None,
cksum=None,
modtime=None,
pkgnames=['SUNWsystem-library'],
)
self.assertEqual(expected, spi._ParseIpsPkgContentsLine(line))
def test_ParseIpsPkgContentsLineLink(self):
spi = system_pkgmap.Indexer()
line = PKGCONTENT_LINE_1
expected = representations.PkgmapEntry(
line=PKGCONTENT_LINE_1,
class_=None,
mode=None,
owner=None,
group=None,
type_='s',
major=None,
minor=None,
size=None,
path='/bin',
target='./usr/bin',
cksum=None,
modtime=None,
pkgnames=['SUNWsystem-core-os'],
)
self.assertEqual(expected, spi._ParseIpsPkgContentsLine(line))
def test_ParseIpsPkgContentsLineDir(self):
spi = system_pkgmap.Indexer()
line = PKGCONTENT_LINE_2
expected = representations.PkgmapEntry(
line=PKGCONTENT_LINE_2,
class_=None,
mode='0755',
owner='root',
group='sys',
path='/dev',
target=None,
type_='d',
major=None,
minor=None,
size=None,
cksum=None,
modtime=None,
pkgnames=['SUNWsystem-core-os'],
)
self.assertEqual(expected, spi._ParseIpsPkgContentsLine(line))
def test_ParseSrv4PkgmapLineTypeSymlink(self):
spi = system_pkgmap.Indexer()
expected = {
'cksum': None,
'class_': 'none',
'major': None,
'minor': None,
'group': None,
'mode': None,
'modtime': None,
'owner': None,
'path': '/opt/csw/apache2/lib/libapr-1.so.0',
'pkgnames': ['CSWapache2rt'],
'size': None,
'target': 'libapr-1.so.0.3.8',
'type_': 's',
'line': PKGMAP_LINE_3,
}
self.assertEqual(expected,
spi._ParseSrv4PkgmapLine(PKGMAP_LINE_3)._asdict())
def test_ParseSrv4PkgmapLineFile(self):
spi = system_pkgmap.Indexer()
expected = {
'cksum': '28258',
'class_': 'none',
'group': 'bin',
'major': None,
'minor': None,
'mode': '0755',
'modtime': '1018129099',
'owner': 'root',
'path': '/usr/lib/sparcv9/libpthread.so.1',
'pkgnames': ['SUNWcslx'],
'size': '41296',
'target': None,
'type_': 'f',
'line': PKGMAP_LINE_1
}
self.assertEqual(expected,
spi._ParseSrv4PkgmapLine(PKGMAP_LINE_1)._asdict())
def test_ParseSrv4PkgmapLineTypePipe(self):
"""A pipe is a valid type and we have to cope with it."""
spi = system_pkgmap.Indexer()
expected = {
'modtime': None,
'major': None,
'pkgnames': ['SUNWscn-agentfacade-r'],
'cksum': None,
'owner': 'root',
'path': '/etc/scn/scn_aa_read',
'line': PKGMAP_LINE_8,
'class_': 'none',
'size': None,
'group': 'sys',
'target': None,
'mode': '0600',
'type_': 'p',
'minor': None,
}
self.assertEqual(expected,
spi._ParseSrv4PkgmapLine(PKGMAP_LINE_8)._asdict())
def test_ParseSrv4PkgmapLineTypeQuestionMark(self):
"""A question mark is not a valid type, but we have to cope with it."""
spi = system_pkgmap.Indexer()
expected = {
'modtime': None,
'major': None,
'pkgnames': ['CSWgcc3g77', 'CSWgcc3core'],
'cksum': None,
'owner': None,
'path': '/opt/csw/gcc3/lib/gcc/sparc-sun-solaris2.8/3.4.6/include',
'line': PKGMAP_LINE_5,
'class_': 'none',
'size': None,
'group': None,
'target': None,
'mode': None,
'type_': 'd',
'minor': None,
}
self.assertEqual(expected,
spi._ParseSrv4PkgmapLine(PKGMAP_LINE_5)._asdict())
def test_ParseSrv4PkgmapLibc(self):
"""A question mark is not a valid type, but we have to cope with it."""
spi = system_pkgmap.Indexer()
expected = {
'modtime': '1250803966',
'major': None,
'pkgnames': ['SUNWcsl'],
'cksum': '58567',
'owner': 'root',
'path': '/usr/lib/libc.so.1',
'line':
'/usr/lib/libc.so.1 f none 0755 root bin 867444 58567 1250803966 SUNWcsl',
'class_': 'none',
'size': '867444',
'group': 'bin',
'target': None,
'mode': '0755',
'type_': 'f',
'minor': None,
}
self.assertEqual(expected,
spi._ParseSrv4PkgmapLine(PKGMAP_LINE_6)._asdict())
def test_IpsNameToSrv4Name(self):
spi = system_pkgmap.Indexer()
self.assertEqual('SUNWsystem-core-os',
spi._IpsNameToSrv4Name("system/core-os"))
def test_ParseIpsPkgListLine(self):
spi = system_pkgmap.Indexer()
expected = ('SUNWdeveloper-versioning-sccs',
'Source Code Control System')
self.assertEqual(expected, spi._ParseIpsPkgListLine(PKGLIST_LINE_1))
def test_ParseSrv4PkgmapExclamationMark(self):
spi = system_pkgmap.Indexer()
self.assertEqual(["!CSWmozilla"],
spi._ParseSrv4PkgmapLine(PKGMAP_LINE_7).pkgnames)
def Disabledtest_GetDataStructure(self):
spi = system_pkgmap.Indexer()
expected = {
'arch':
'sparc',
'osrel':
'SunOS5.11',
'contents': [{
'cksum': '28258',
'class': 'none',
'group': 'bin',
'line':
'/usr/lib/sparcv9/libpthread.so.1 f none 0755 root bin 41296 28258 1018129099 SUNWcslx',
'major': None,
'minor': None,
'mode': '0755',
'modtime': '1018129099',
'owner': 'root',
'path': '/usr/lib/sparcv9/libpthread.so.1',
'pkgnames': ['SUNWcslx'],
'size': '41296',
'target': None,
'type': 'f'
}, {
'cksum': '6287',
'class': 'none',
'group': 'bin',
'line':
'/usr/lib/libCrun.so.1 f none 0755 root bin 63588 6287 1256043984 SUNWlibC',
'major': None,
'minor': None,
'mode': '0755',
'modtime': '1256043984',
'owner': 'root',
'path': '/usr/lib/libCrun.so.1',
'pkgnames': ['SUNWlibC'],
'size': '63588',
'target': None,
'type': 'f'
}, {
'group': None,
'line': 'bin\tlink\tsystem/core-os\t./usr/bin',
'mode': None,
'owner': None,
'path': '/bin',
'pkgnames': ['SUNWsystem-core-os'],
'target': './usr/bin',
'type': 's'
}, {
'group': 'sys',
'line': 'dev\tdir\tsystem/core-os\t\t0755\troot\tsys',
'mode': '0755',
'owner': 'root',
'path': '/dev',
'pkgnames': ['SUNWsystem-core-os'],
'target': None,
'type': 'd'
}],
'pkginfo': {
'SUNWdeveloper-solarisstudio-122-c': u'C++ Compilers',
'SUNWdeveloper-versioning-sccs': u'Source Code Control System',
'SUNWpcan': u'Cisco-Aironet 802.11b driver',
'SUNWwpau': u'Wireless WPA Supplicant, (Usr)'
}
}
srv4_pkginfos_stream = (
PKGINFO_LINE_1,
PKGINFO_LINE_2,
)
ips_pkginfos_stream = (
PKGLIST_LINE_1,
PKGLIST_LINE_2,
)
srv4_pkgcontents_stream = (
PKGMAP_LINE_1,
PKGMAP_LINE_2,
)
ips_pkgcontents_stream = (
PKGCONTENT_LINE_1,
PKGCONTENT_LINE_2,
)
self.assertEqual(
expected,
spi.GetDataStructure(srv4_pkgcontents_stream, srv4_pkginfos_stream,
ips_pkgcontents_stream, ips_pkginfos_stream,
'SunOS5.11', 'sparc', False))
def test_ParseSrv4PkgmapLineTypeWrongSyntax(self):
spi = system_pkgmap.Indexer()
self.assertRaises(system_pkgmap.ParsingError, spi._ParseSrv4PkgmapLine,
"/")
def main():
parser = optparse.OptionParser(USAGE)
parser.add_option("-d",
"--debug",
dest="debug",
default=False,
action="store_true",
help="Turn on debugging messages")
parser.add_option(
"-t",
"--pkg-review-template",
dest="pkg_review_template",
help="A Cheetah template used for package review reports.")
parser.add_option("-r",
"--os-release",
dest="osrel",
default="SunOS5.10",
help="E.g. SunOS5.10")
parser.add_option("-a",
"--arch",
dest="arch",
default="sparc",
help="'i386' or 'sparc'")
parser.add_option("-c",
"--catalog-release",
dest="catrel",
default="unstable",
help="E.g. unstable, dublin")
parser.add_option("--replace",
dest="replace",
default=False,
action="store_true",
help="Replace packages when importing (importpkg)")
parser.add_option("--profile",
dest="profile",
default=False,
action="store_true",
help="Turn on profiling")
parser.add_option("--force-unpack",
dest="force_unpack",
default=False,
action="store_true",
help="Force unpacking of packages")
options, args = parser.parse_args()
logging_level = logging.INFO
if options.debug:
logging_level = logging.DEBUG
fmt = '%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s'
logging.basicConfig(format=fmt, level=logging_level)
if not args:
raise UsageError("Please specify a command. See --help.")
# SetUpSqlobjectConnection needs to be called after
# logging.basicConfig
configuration.SetUpSqlobjectConnection()
command = args[0]
args = args[1:]
if command == 'show':
subcommand = args[0]
args = args[1:]
elif command == 'pkg':
subcommand = args[0]
args = args[1:]
else:
subcommand = None
md5_sums = args
if (command, subcommand) == ('show', 'errors'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
res = m.CheckpkgErrorTag.select(
m.CheckpkgErrorTag.q.srv4_file == srv4)
for row in res:
print 'overridden' if row.overridden else 'active',
print row.pkgname, row.tag_name, row.tag_info, row.catrel.name, row.arch.name,
print row.os_rel.short_name
elif (command, subcommand) == ('show', 'overrides'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
res = m.CheckpkgOverride.select(
m.CheckpkgOverride.q.srv4_file == srv4)
for row in res:
print row.pkgname, row.tag_name, row.tag_info
elif (command, subcommand) == ('show', 'pkg'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
t = Template(SHOW_PKG_TMPL, searchList=[srv4])
sys.stdout.write(unicode(t))
elif command == 'gen-html':
config = configuration.GetConfig()
username, password = rest.GetUsernameAndPassword()
rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get(
'rest', 'releases'),
username=username,
password=password,
debug=options.debug)
g = HtmlGenerator(md5_sums, options.pkg_review_template, rest_client,
options.debug)
sys.stdout.write(g.GenerateHtml())
elif command == 'initdb':
config = configuration.GetConfig()
database.InitDB(config)
elif command == 'importpkg':
collector = package_stats.StatsCollector(logger=logging,
debug=options.debug)
file_list = args
catalog_entries = []
for file_name in file_list:
file_hash = hashlib.md5()
chunk_size = 2 * 1024 * 1024
with open(file_name, 'rb') as fd:
data = fd.read(chunk_size)
while data:
file_hash.update(data)
data = fd.read(chunk_size)
data_md5_sum = file_hash.hexdigest()
catalog_entry = {
'md5sum': data_md5_sum,
'file_basename': os.path.basename(file_name),
'pkg_path': file_name,
}
catalog_entries.append(catalog_entry)
md5_list = collector.CollectStatsFromCatalogEntries(
catalog_entries, force_unpack=options.force_unpack)
config = configuration.GetConfig()
rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get(
'rest', 'releases'),
debug=options.debug)
for md5_sum in md5_list:
logging.debug("Importing %s", md5_sum)
rest_client.RegisterLevelTwo(md5_sum)
elif command == 'removepkg':
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
in_catalogs = list(srv4.in_catalogs)
if in_catalogs:
for in_catalog in in_catalogs:
logging.warning("%s", in_catalog)
logging.warning(
"Not removing from the database, because the package "
"in question is part of at least one catalog.")
else:
logging.info("Removing %s", srv4)
srv4.DeleteAllDependentObjects()
srv4.destroySelf()
elif command == 'add-to-cat':
if len(args) < 4:
raise UsageError("Not enough arguments, see usage.")
user = getpass.getuser()
osrel, arch, catrel = args[:3]
username, password = rest.GetUsernameAndPassword()
rest_client = rest.RestClient(username=username, password=password)
md5_sums = args[3:]
for md5_sum in md5_sums:
rest_client.AddSvr4ToCatalog(catrel, arch, osrel, md5_sum)
elif command == 'del-from-cat':
if len(args) < 4:
raise UsageError("Not enough arguments, see usage.")
osrel, arch, catrel = args[:3]
md5_sums = args[3:]
username, password = rest.GetUsernameAndPassword()
rest_client = rest.RestClient(username=username, password=password)
for md5_sum in md5_sums:
rest_client.RemoveSvr4FromCatalog(catrel, arch, osrel, md5_sum)
elif command == 'system-metadata-to-disk':
logging.debug("Args: %s", args)
outfile = None
infile_contents = common_constants.DEFAULT_INSTALL_CONTENTS_FILE
infile_pkginfo = None
osrel, arch = (None, None)
if len(args) >= 2:
infile_contents = args[0]
infile_pkginfo = args[1]
if len(args) >= 3:
outfile = args[2]
if len(args) >= 4:
if len(args) == 5:
osrel, arch = args[3:5]
else:
raise UsageError("Wrong number of arguments (%s), see usage." %
len(args))
spi = system_pkgmap.Indexer(outfile, infile_contents, infile_pkginfo,
osrel, arch)
spi.IndexAndSave()
elif command == 'import-system-metadata':
if len(args) < 2:
raise UsageError(
"Usage: ... import-system-metadata <osrel> <arch>")
osrel = args[0]
arch = args[1]
importer = system_pkgmap.InstallContentsImporter(osrel,
arch,
debug=options.debug)
importer.Import(show_progress=(not options.debug))
elif (command, subcommand) == ('pkg', 'search'):
logging.debug("Searching for %s", args)
sqo_osrel = m.OsRelease.selectBy(short_name=options.osrel).getOne()
sqo_arch = m.Architecture.selectBy(name=options.arch).getOne()
sqo_catrel = m.CatalogRelease.selectBy(name=options.catrel).getOne()
if len(args) < 1:
logging.fatal("Wrong number of arguments: %s", len(args))
raise SystemExit
for catalogname in args:
join = [
sqlbuilder.INNERJOINOn(
None, m.Srv4FileInCatalog,
m.Srv4FileInCatalog.q.srv4file == m.Srv4FileStats.q.id),
]
res = m.Srv4FileStats.select(
sqlobject.AND(
m.Srv4FileInCatalog.q.osrel == sqo_osrel,
m.Srv4FileInCatalog.q.arch == sqo_arch,
m.Srv4FileInCatalog.q.catrel == sqo_catrel,
m.Srv4FileStats.q.catalogname.contains(catalogname),
m.Srv4FileStats.q.use_to_generate_catalogs == True),
join=join,
).orderBy("catalogname")
for sqo_srv4 in res:
print "%s %s" % (sqo_srv4.basename, sqo_srv4.md5_sum)
elif command == 'sync-cat-from-file':
if len(args) != 4:
raise UsageError("Wrong number of arguments, see usage.")
osrel, arch, catrel, catalog_file = args
ci = CatalogImporter(debug=options.debug)
ci.SyncFromCatalogFile(osrel, arch, catrel, catalog_file)
elif command == 'sync-catalogs-from-tree':
if len(args) != 2:
raise UsageError("Wrong number of arguments, see usage.")
ci = CatalogImporter(debug=options.debug)
catrel, base_dir = args
ci.SyncFromCatalogTree(catrel, base_dir, options.force_unpack)
elif (command, subcommand) == ('show', 'cat'):
sqo_osrel, sqo_arch, sqo_catrel = m.GetSqoTriad(
options.osrel, options.arch, options.catrel)
res = m.GetCatPackagesResult(sqo_osrel, sqo_arch, sqo_catrel)
for obj in res:
print obj.catalogname, obj.basename, obj.md5_sum
elif (command, subcommand) == ('show', 'files'):
md5_sum = args[0]
join = [
sqlbuilder.INNERJOINOn(
None, m.Srv4FileStats,
m.CswFile.q.srv4_file == m.Srv4FileStats.q.id),
]
res = m.CswFile.select(
m.Srv4FileStats.q.md5_sum == md5_sum,
join=join,
)
for obj in res:
print os.path.join(obj.path, obj.basename)
elif (command, subcommand) == ('show', 'basename'):
db_catalog = checkpkg_lib.Catalog()
for arg in args:
pkgs_by_path = db_catalog.GetPathsAndPkgnamesByBasename(
arg, options.osrel, options.arch, options.catrel)
for file_path in pkgs_by_path:
print os.path.join(file_path,
arg), ", ".join(pkgs_by_path[file_path])
elif (command, subcommand) == ('show', 'filename'):
db_catalog = checkpkg_lib.Catalog()
for arg in args:
pkgs = db_catalog.GetPkgByPath(arg, options.osrel, options.arch,
options.catrel)
print " ".join(pkgs)
else:
raise UsageError("Command unrecognized: %s" % command)
def test_ParseSrv4PkginfoLine(self):
spi = system_pkgmap.Indexer()
expected = ('SUNWwpau', 'Wireless WPA Supplicant, (Usr)')
self.assertEqual(expected, spi._ParseSrv4PkginfoLine(PKGINFO_LINE_1))
