def _ImportFiles(self, data, include_prefixes=None, show_progress=False):
logging.debug("_ImportFiles()")
osrel = data["osrel"]
arch = data["arch"]
contents = data["contents"]
catalog = checkpkg_lib.Catalog()
srv4_files_to_catalog = set()
# The progressbar library doesn't like handling larger numbers
# It displays up to 99% if we feed it a maxval in the range of hundreds of
# thousands.
progressbar_divisor = int(len(contents) / 1000)
if progressbar_divisor < 1:
progressbar_divisor = 1
update_period = 1L
count = itertools.count()
if show_progress:
pbar = progressbar.ProgressBar()
else:
pbar = mute_progressbar.MuteProgressBar()
pbar.maxval = len(contents) / progressbar_divisor
pbar.start()
cleaned_pkgs = set()
for d in contents:
i = count.next()
if not i % update_period and (i / progressbar_divisor) <= pbar.maxval:
pbar.update(i / progressbar_divisor)
for pkgname in d["pkgnames"]:
pkgname = self.SanitizeInstallContentsPkgname(pkgname)
# If a package is a packge of our own,
# it should not be imported that way; own packages should be
# only managed by adding them to specific catalogs.
skip_pkgname = False
for prefix in common_constants.OWN_PKGNAME_PREFIXES:
if pkgname.startswith(prefix):
skip_pkgname = True
break
# Prefix whilelist - whitelisted prefixes win.
if include_prefixes:
for prefix_to_include in include_prefixes:
if pkgname.startswith(prefix_to_include):
skip_pkgname = False
break
if skip_pkgname:
continue
# We'll create one file instance for each package
try:
sqo_srv4 = self._GetFakeSrv4(pkgname, osrel, arch)
except sqlobject.main.SQLObjectNotFound, e:
print d
raise
if sqo_srv4 not in cleaned_pkgs:
sqo_srv4.RemoveAllCswFiles()
cleaned_pkgs.add(sqo_srv4)
sqo_pkginst = self._GetPkginst(pkgname)
f_path, f_basename = os.path.split(d["path"])
# This is really slow (one run ~1h), but works.
# To speed it up, raw SQL + cursor.executemany() could be used, but
# there's a incompatibility between MySQL and sqlite drivers:
# MySQL: INSERT ... VALUES (%s, %s, %s);
# sqlite: INSERT ... VALUES (?, ?, ?);
# For now, using the sqlobject ORM which is slow, but at least
# handles compatibility issues.
csw_file = m.CswFile(pkginst=sqo_pkginst,
line=d["line"], path=f_path, basename=f_basename,
srv4_file=sqo_srv4)
srv4_files_to_catalog.add(sqo_srv4)
line_u = pkgmap_entry["line"].decode("utf-8")
f_path, basename = os.path.split(
pkgmap_entry["path"].decode('utf-8'))
except UnicodeDecodeError as e:
line_u = pkgmap_entry["line"].decode("latin1")
f_path, basename = os.path.split(
pkgmap_entry["path"].decode('latin1'))
except UnicodeEncodeError as e:
# the line was already in unicode
line_u = pkgmap_entry['line']
f_path, basename = os.path.split(pkgmap_entry["path"])
# If this fails too, code change will be needed.
f = m.CswFile(basename=basename,
path=f_path,
line=line_u,
pkginst=pkginst,
srv4_file=stats)
# Save dependencies in the database. First remove any dependency rows
# that might be in the database.
# TODO(maciej): Unit test it
deps_res = m.Srv4DependsOn.select(m.Srv4DependsOn.q.srv4_file == stats)
for dep_obj in deps_res:
dep_obj.destroySelf()
for dep_pkgname, unused_desc in pkg_stats["depends"]:
dep_pkginst = cls.GetOrSetPkginst(dep_pkgname)
obj = m.Srv4DependsOn(srv4_file=stats, pkginst=dep_pkginst)
# At this point, we've registered the srv4 file.
# Setting the registered bit to True
stats.registered = True