def db_update(archname, reponame, pkgs, force=False):
"""
Parses a list of packages and updates the packages database accordingly.
"""
logger.info('Updating %s (%s)', reponame, archname)
dbpkgs = update_common(archname, reponame, pkgs, True)
repository = Repo.objects.get(name__iexact=reponame)
architecture = Arch.objects.get(name=archname)
# This makes our inner loop where we find packages by name *way* more
# efficient by not having to go to the database for each package to
# SELECT them by name.
dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs}
dbset = set(dbdict.keys())
syncset = {pkg.name for pkg in pkgs}
in_sync_not_db = syncset - dbset
logger.info("%d packages in sync not db", len(in_sync_not_db))
# packages in syncdb and not in database (add to database)
for pkg in (pkg for pkg in pkgs if pkg.name in in_sync_not_db):
logger.info("Adding package %s", pkg.name)
timestamp = now()
dbpkg = Package(pkgname=pkg.name,
arch=architecture,
repo=repository,
created=timestamp)
try:
with transaction.atomic():
populate_pkg(dbpkg, pkg, timestamp=timestamp)
Update.objects.log_update(None, dbpkg)
if not Package.objects.filter(pkgname=pkg.name).exclude(
id=dbpkg.id).exists():
if not User.objects.filter(
package_relations__pkgbase=dbpkg.pkgbase,
package_relations__type=PackageRelation.MAINTAINER
).exists():
packager = finder.find(pkg.packager)
if packager:
prel = PackageRelation(
pkgbase=dbpkg.pkgbase,
user=packager,
type=PackageRelation.MAINTAINER)
prel.save()
except IntegrityError:
if architecture.agnostic:
logger.warning(
"Could not add package %s; "
"not fatal if another thread beat us to it.", pkg.name)
else:
logger.exception("Could not add package %s", pkg.name)
# packages in database and not in syncdb (remove from database)
for pkgname in (dbset - syncset):
logger.info("Removing package %s", pkgname)
dbpkg = dbdict[pkgname]
with transaction.atomic():
Update.objects.log_update(dbpkg, None)
# no race condition here as long as simultaneous threads both
# issue deletes; second delete will be a no-op
delete_pkg_files(dbpkg)
# Delete all matching FlagRequests when removing a package from a repo.
requests = FlagRequest.objects.filter(pkgbase=dbpkg.pkgbase,
repo=dbpkg.repo)
requests.delete()
dbpkg.delete()
# packages in both database and in syncdb (update in database)
pkg_in_both = syncset & dbset
for pkg in (x for x in pkgs if x.name in pkg_in_both):
logger.debug("Checking package %s", pkg.name)
dbpkg = dbdict[pkg.name]
timestamp = None
# for a force, we don't want to update the timestamp.
# for a non-force, we don't want to do anything at all.
if not force and pkg_same_version(pkg, dbpkg):
continue
elif not force:
timestamp = now()
# The odd select_for_update song and dance here are to ensure
# simultaneous updates don't happen on a package, causing
# files/depends/all related items to be double-imported.
with transaction.atomic():
dbpkg = Package.objects.select_for_update().get(id=dbpkg.id)
if not force and pkg_same_version(pkg, dbpkg):
logger.debug("Package %s was already updated", pkg.name)
continue
logger.info("Updating package %s", pkg.name)
prevpkg = copy(dbpkg)
populate_pkg(dbpkg, pkg, force=force, timestamp=timestamp)
Update.objects.log_update(prevpkg, dbpkg)
logger.info('Finished updating arch: %s', archname)
def db_update(archname, reponame, pkgs, force=False):
"""
Parses a list of packages and updates the packages database accordingly.
"""
logger.info('Updating %s (%s)', reponame, archname)
dbpkgs = update_common(archname, reponame, pkgs, True)
repository = Repo.objects.get(name__iexact=reponame)
architecture = Arch.objects.get(name=archname)
# This makes our inner loop where we find packages by name *way* more
# efficient by not having to go to the database for each package to
# SELECT them by name.
dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs}
dbset = set(dbdict.keys())
syncset = {pkg.name for pkg in pkgs}
in_sync_not_db = syncset - dbset
logger.info("%d packages in sync not db", len(in_sync_not_db))
# packages in syncdb and not in database (add to database)
for pkg in (pkg for pkg in pkgs if pkg.name in in_sync_not_db):
logger.info("Adding package %s", pkg.name)
timestamp = now()
dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository,
created=timestamp)
try:
with transaction.atomic():
populate_pkg(dbpkg, pkg, timestamp=timestamp)
Update.objects.log_update(None, dbpkg)
if not Package.objects.filter(
pkgname=pkg.name).exclude(id=dbpkg.id).exists():
if not User.objects.filter(
package_relations__pkgbase=dbpkg.pkgbase,
package_relations__type=PackageRelation.MAINTAINER
).exists():
packager = finder.find(pkg.packager)
if packager:
prel = PackageRelation(pkgbase=dbpkg.pkgbase,
user=packager,
type=PackageRelation.MAINTAINER)
prel.save()
except IntegrityError:
if architecture.agnostic:
logger.warning("Could not add package %s; "
"not fatal if another thread beat us to it.",
pkg.name)
else:
logger.exception("Could not add package %s", pkg.name)
# packages in database and not in syncdb (remove from database)
for pkgname in (dbset - syncset):
logger.info("Removing package %s", pkgname)
dbpkg = dbdict[pkgname]
with transaction.atomic():
Update.objects.log_update(dbpkg, None)
# no race condition here as long as simultaneous threads both
# issue deletes; second delete will be a no-op
delete_pkg_files(dbpkg)
# Delete all matching FlagRequests when removing a package from a repo.
requests = FlagRequest.objects.filter(pkgbase=dbpkg.pkgbase, repo=dbpkg.repo)
requests.delete()
dbpkg.delete()
# packages in both database and in syncdb (update in database)
pkg_in_both = syncset & dbset
for pkg in (x for x in pkgs if x.name in pkg_in_both):
logger.debug("Checking package %s", pkg.name)
dbpkg = dbdict[pkg.name]
timestamp = None
# for a force, we don't want to update the timestamp.
# for a non-force, we don't want to do anything at all.
if not force and pkg_same_version(pkg, dbpkg):
continue
elif not force:
timestamp = now()
# The odd select_for_update song and dance here are to ensure
# simultaneous updates don't happen on a package, causing
# files/depends/all related items to be double-imported.
with transaction.atomic():
dbpkg = Package.objects.select_for_update().get(id=dbpkg.id)
if not force and pkg_same_version(pkg, dbpkg):
logger.debug("Package %s was already updated", pkg.name)
continue
logger.info("Updating package %s", pkg.name)
prevpkg = copy(dbpkg)
populate_pkg(dbpkg, pkg, force=force, timestamp=timestamp)
Update.objects.log_update(prevpkg, dbpkg)
logger.info('Finished updating arch: %s', archname)