def test_reload(self):
# reload() loads the given objects using queries generated by
# gen_reload_queries().
db_object = self.factory.makeComponent()
db_object_naked = proxy.removeSecurityProxy(db_object)
db_object_info = get_obj_info(db_object_naked)
IStore(db_object).flush()
self.failUnlessEqual(None, db_object_info.get('invalidated'))
IStore(db_object).invalidate(db_object)
self.failUnlessEqual(True, db_object_info.get('invalidated'))
bulk.reload([db_object])
self.failUnlessEqual(None, db_object_info.get('invalidated'))
def test_reload(self):
# reload() loads the given objects using queries generated by
# gen_reload_queries().
db_object = self.factory.makeComponent()
db_object_naked = proxy.removeSecurityProxy(db_object)
db_object_info = get_obj_info(db_object_naked)
IStore(db_object).flush()
self.failUnlessEqual(None, db_object_info.get('invalidated'))
IStore(db_object).invalidate(db_object)
self.failUnlessEqual(True, db_object_info.get('invalidated'))
bulk.reload([db_object])
self.failUnlessEqual(None, db_object_info.get('invalidated'))
def updateBugWatches(self, remotesystem, bug_watches_to_update, now=None,
batch_size=None):
"""Update the given bug watches."""
# Save the url for later, since we might need it to report an
# error after a transaction has been aborted.
bug_tracker_url = remotesystem.baseurl
# Some tests pass a list of bug watches whilst checkwatches.py
# will pass a SelectResults instance. We convert bug_watches to a
# list here to ensure that were're doing sane things with it
# later on.
with self.transaction:
bug_watches = list(bug_watches_to_update)
bug_watch_ids = [bug_watch.id for bug_watch in bug_watches]
# Fetch the time on the server. We'll use this in
# _getRemoteIdsToCheck() and when determining whether we can
# sync comments or not.
with record_errors(self.transaction, bug_watch_ids):
server_time = remotesystem.getCurrentDBTime()
remote_ids = self._getRemoteIdsToCheck(
remotesystem, bug_watches, server_time, now, batch_size)
remote_ids_to_check = remote_ids['remote_ids_to_check']
all_remote_ids = remote_ids['all_remote_ids']
unmodified_remote_ids = remote_ids['unmodified_remote_ids']
# Remove from the list of bug watches any watch whose remote ID
# doesn't appear in the list of IDs to check.
with self.transaction:
reload(bug_watches)
for bug_watch in list(bug_watches):
if bug_watch.remotebug not in remote_ids_to_check:
bug_watches.remove(bug_watch)
self.logger.info(
"Updating %i watches for %i bugs on %s" % (
len(bug_watches), len(remote_ids_to_check), bug_tracker_url))
with record_errors(self.transaction, bug_watch_ids):
remotesystem.initializeRemoteBugDB(remote_ids_to_check)
for remote_bug_id in all_remote_ids:
remote_bug_updater = self.remote_bug_updater_factory(
self, remotesystem, remote_bug_id, bug_watch_ids,
unmodified_remote_ids, server_time)
remote_bug_updater.updateRemoteBug()
def updateBugWatches(self, remotesystem, bug_watches_to_update, now=None,
batch_size=None):
"""Update the given bug watches."""
# Save the url for later, since we might need it to report an
# error after a transaction has been aborted.
bug_tracker_url = remotesystem.baseurl
# Some tests pass a list of bug watches whilst checkwatches.py
# will pass a SelectResults instance. We convert bug_watches to a
# list here to ensure that were're doing sane things with it
# later on.
with self.transaction:
bug_watches = list(bug_watches_to_update)
bug_watch_ids = [bug_watch.id for bug_watch in bug_watches]
# Fetch the time on the server. We'll use this in
# _getRemoteIdsToCheck() and when determining whether we can
# sync comments or not.
with record_errors(self.transaction, bug_watch_ids):
server_time = remotesystem.getCurrentDBTime()
remote_ids = self._getRemoteIdsToCheck(
remotesystem, bug_watches, server_time, now, batch_size)
remote_ids_to_check = remote_ids['remote_ids_to_check']
all_remote_ids = remote_ids['all_remote_ids']
unmodified_remote_ids = remote_ids['unmodified_remote_ids']
# Remove from the list of bug watches any watch whose remote ID
# doesn't appear in the list of IDs to check.
with self.transaction:
reload(bug_watches)
for bug_watch in list(bug_watches):
if bug_watch.remotebug not in remote_ids_to_check:
bug_watches.remove(bug_watch)
self.logger.info(
"Updating %i watches for %i bugs on %s" % (
len(bug_watches), len(remote_ids_to_check), bug_tracker_url))
with record_errors(self.transaction, bug_watch_ids):
remotesystem.initializeRemoteBugDB(remote_ids_to_check)
for remote_bug_id in all_remote_ids:
remote_bug_updater = self.remote_bug_updater_factory(
self, remotesystem, remote_bug_id, bug_watch_ids,
unmodified_remote_ids, server_time)
remote_bug_updater.updateRemoteBug()
def test_source_overrides_constant_query_count(self):
# The query count is constant, no matter how many sources are
# checked.
spns = []
distroseries = self.factory.makeDistroSeries()
pocket = self.factory.getAnyPocket()
for i in xrange(10):
spph = self.factory.makeSourcePackagePublishingHistory(
distroseries=distroseries, archive=distroseries.main_archive,
pocket=pocket)
spns.append(spph.sourcepackagerelease.sourcepackagename)
flush_database_caches()
distroseries.main_archive
bulk.reload(spns)
policy = FromExistingOverridePolicy()
with StormStatementRecorder() as recorder:
policy.calculateSourceOverrides(
spph.distroseries.main_archive, spph.distroseries,
spph.pocket, spns)
self.assertThat(recorder, HasQueryCount(Equals(4)))
def test_binary_overrides_constant_query_count(self):
# The query count is constant, no matter how many bpn-das pairs are
# checked.
bpns = []
distroarchseries = self.factory.makeDistroArchSeries()
distroseries = distroarchseries.distroseries
distroseries.nominatedarchindep = distroarchseries
pocket = self.factory.getAnyPocket()
for i in xrange(10):
bpph = self.factory.makeBinaryPackagePublishingHistory(
distroarchseries=distroarchseries,
archive=distroseries.main_archive, pocket=pocket)
bpns.append((bpph.binarypackagerelease.binarypackagename, None))
flush_database_caches()
distroseries.main_archive
bulk.reload(bpn[0] for bpn in bpns)
policy = FromExistingOverridePolicy()
with StormStatementRecorder() as recorder:
policy.calculateBinaryOverrides(
distroseries.main_archive, distroseries, pocket, bpns)
self.assertThat(recorder, HasQueryCount(Equals(4)))
def _getRemoteIdsToCheck(self, remotesystem, bug_watches,
server_time=None, now=None, batch_size=None):
"""Return the remote bug IDs to check for a set of bug watches.
The remote bug tracker is queried to find out which of the
remote bugs in `bug_watches` have changed since they were last
checked. Those which haven't changed are excluded.
:param bug_watches: A set of `BugWatch`es to be checked.
:param remotesystem: The `ExternalBugtracker` on which
`getModifiedRemoteBugs`() should be called
:param server_time: The time according to the remote server.
This may be None when the server doesn't specify a remote time.
:param now: The current time (used for testing)
:return: A list of remote bug IDs to be updated.
"""
# Check that the remote server's notion of time agrees with
# ours. If not, raise a TooMuchTimeSkew error, since if the
# server's wrong about the time it'll mess up all our times when
# we import things.
if now is None:
now = datetime.now(pytz.timezone('UTC'))
if (server_time is not None and
abs(server_time - now) > ACCEPTABLE_TIME_SKEW):
raise TooMuchTimeSkew(abs(server_time - now))
# We limit the number of watches we're updating by the
# ExternalBugTracker's batch_size. In an ideal world we'd just
# slice the bug_watches list but for the sake of testing we need
# to ensure that the list of bug watches is ordered by remote
# bug id before we do so.
if batch_size is None:
# If a batch_size hasn't been passed, use the one specified
# by the ExternalBugTracker.
batch_size = remotesystem.batch_size
with self.transaction:
reload(bug_watches)
old_bug_watches = set(
bug_watch for bug_watch in bug_watches
if bug_watch.lastchecked is not None)
if len(old_bug_watches) == 0:
oldest_lastchecked = None
else:
oldest_lastchecked = min(
bug_watch.lastchecked for bug_watch in old_bug_watches)
# Adjust for possible time skew, and some more, just to be
# safe.
oldest_lastchecked -= (
ACCEPTABLE_TIME_SKEW + timedelta(minutes=1))
# Collate the remote IDs.
remote_old_ids = sorted(
set(bug_watch.remotebug for bug_watch in old_bug_watches))
remote_new_ids = sorted(
set(bug_watch.remotebug for bug_watch in bug_watches
if bug_watch not in old_bug_watches))
# If the remote system is not configured to sync comments,
# don't bother checking for any to push.
if remotesystem.sync_comments:
remote_ids_with_comments = sorted(
bug_watch.remotebug for bug_watch in bug_watches
if bug_watch.unpushed_comments.any() is not None)
else:
remote_ids_with_comments = []
# We only make the call to getModifiedRemoteBugs() if there
# are actually some bugs that we're interested in so as to
# avoid unnecessary network traffic.
if server_time is not None and len(remote_old_ids) > 0:
if batch_size == BATCH_SIZE_UNLIMITED:
remote_old_ids_to_check = (
remotesystem.getModifiedRemoteBugs(
remote_old_ids, oldest_lastchecked))
else:
# Don't ask the remote system about more than
# batch_size bugs at once, but keep asking until we
# run out of bugs to ask about or we have batch_size
# bugs to check.
remote_old_ids_to_check = []
for index in range(0, len(remote_old_ids), batch_size):
remote_old_ids_to_check.extend(
remotesystem.getModifiedRemoteBugs(
remote_old_ids[index:index + batch_size],
oldest_lastchecked))
if len(remote_old_ids_to_check) >= batch_size:
break
else:
remote_old_ids_to_check = remote_old_ids
# We'll create our remote_ids_to_check list so that it's
# prioritized. We include remote IDs in priority order:
# 1. IDs with comments.
# 2. IDs that haven't been checked.
# 3. Everything else.
remote_ids_to_check = chain(
remote_ids_with_comments, remote_new_ids, remote_old_ids_to_check)
if batch_size != BATCH_SIZE_UNLIMITED:
# Some remote bug IDs may appear in more than one list so
# we must filter the list before slicing.
remote_ids_to_check = islice(
unique(remote_ids_to_check), batch_size)
# Stuff the IDs in a set.
remote_ids_to_check = set(remote_ids_to_check)
# Make sure that unmodified_remote_ids only includes IDs that
# could have been checked but which weren't modified on the
# remote server and which haven't been listed for checking
# otherwise (i.e. because they have comments to be pushed).
unmodified_remote_ids = set(remote_old_ids)
unmodified_remote_ids.difference_update(remote_old_ids_to_check)
unmodified_remote_ids.difference_update(remote_ids_to_check)
all_remote_ids = remote_ids_to_check.union(unmodified_remote_ids)
return {
'remote_ids_to_check': sorted(remote_ids_to_check),
'all_remote_ids': sorted(all_remote_ids),
'unmodified_remote_ids': sorted(unmodified_remote_ids),
}
def _getExternalBugTrackersAndWatches(self, bug_tracker, bug_watches):
"""Return an `ExternalBugTracker` instance for `bug_tracker`."""
with self.transaction:
num_watches = bug_tracker.watches.count()
remotesystem = (
externalbugtracker.get_external_bugtracker(bug_tracker))
# We special-case the Gnome Bugzilla.
is_gnome_bugzilla = bug_tracker == (
getUtility(ILaunchpadCelebrities).gnome_bugzilla)
# Probe the remote system for additional capabilities.
remotesystem_to_use = remotesystem.getExternalBugTrackerToUse()
# Try to hint at how many bug watches to check each time.
suggest_batch_size(remotesystem_to_use, num_watches)
if (is_gnome_bugzilla and remotesystem_to_use.sync_comments):
# If there are no products to sync comments for, disable
# comment sync and return.
if len(self._syncable_gnome_products) == 0:
remotesystem_to_use.sync_comments = False
return [
(remotesystem_to_use, bug_watches),
]
syncable_watches = []
other_watches = []
with self.transaction:
reload(bug_watches)
remote_bug_ids = [
bug_watch.remotebug for bug_watch in bug_watches]
remote_products = (
remotesystem_to_use.getProductsForRemoteBugs(
remote_bug_ids))
with self.transaction:
reload(bug_watches)
for bug_watch in bug_watches:
if (remote_products.get(bug_watch.remotebug) in
self._syncable_gnome_products):
syncable_watches.append(bug_watch)
else:
other_watches.append(bug_watch)
# For bug watches on remote bugs that are against products
# in the _syncable_gnome_products list - i.e. ones with which
# we want to sync comments - we return a BugzillaAPI
# instance with sync_comments=True, otherwise we return a
# similar BugzillaAPI instance, but with sync_comments=False.
remotesystem_for_syncables = remotesystem_to_use
remotesystem_for_others = copy(remotesystem_to_use)
remotesystem_for_others.sync_comments = False
return [
(remotesystem_for_syncables, syncable_watches),
(remotesystem_for_others, other_watches),
]
else:
return [
(remotesystem_to_use, bug_watches),
]
def _getRemoteIdsToCheck(self, remotesystem, bug_watches,
server_time=None, now=None, batch_size=None):
"""Return the remote bug IDs to check for a set of bug watches.
The remote bug tracker is queried to find out which of the
remote bugs in `bug_watches` have changed since they were last
checked. Those which haven't changed are excluded.
:param bug_watches: A set of `BugWatch`es to be checked.
:param remotesystem: The `ExternalBugtracker` on which
`getModifiedRemoteBugs`() should be called
:param server_time: The time according to the remote server.
This may be None when the server doesn't specify a remote time.
:param now: The current time (used for testing)
:return: A list of remote bug IDs to be updated.
"""
# Check that the remote server's notion of time agrees with
# ours. If not, raise a TooMuchTimeSkew error, since if the
# server's wrong about the time it'll mess up all our times when
# we import things.
if now is None:
now = datetime.now(pytz.timezone('UTC'))
if (server_time is not None and
abs(server_time - now) > ACCEPTABLE_TIME_SKEW):
raise TooMuchTimeSkew(abs(server_time - now))
# We limit the number of watches we're updating by the
# ExternalBugTracker's batch_size. In an ideal world we'd just
# slice the bug_watches list but for the sake of testing we need
# to ensure that the list of bug watches is ordered by remote
# bug id before we do so.
if batch_size is None:
# If a batch_size hasn't been passed, use the one specified
# by the ExternalBugTracker.
batch_size = remotesystem.batch_size
with self.transaction:
reload(bug_watches)
old_bug_watches = set(
bug_watch for bug_watch in bug_watches
if bug_watch.lastchecked is not None)
if len(old_bug_watches) == 0:
oldest_lastchecked = None
else:
oldest_lastchecked = min(
bug_watch.lastchecked for bug_watch in old_bug_watches)
# Adjust for possible time skew, and some more, just to be
# safe.
oldest_lastchecked -= (
ACCEPTABLE_TIME_SKEW + timedelta(minutes=1))
# Collate the remote IDs.
remote_old_ids = sorted(
set(bug_watch.remotebug for bug_watch in old_bug_watches))
remote_new_ids = sorted(
set(bug_watch.remotebug for bug_watch in bug_watches
if bug_watch not in old_bug_watches))
# If the remote system is not configured to sync comments,
# don't bother checking for any to push.
if remotesystem.sync_comments:
remote_ids_with_comments = sorted(
bug_watch.remotebug for bug_watch in bug_watches
if bug_watch.unpushed_comments.any() is not None)
else:
remote_ids_with_comments = []
# We only make the call to getModifiedRemoteBugs() if there
# are actually some bugs that we're interested in so as to
# avoid unnecessary network traffic.
if server_time is not None and len(remote_old_ids) > 0:
if batch_size == BATCH_SIZE_UNLIMITED:
remote_old_ids_to_check = (
remotesystem.getModifiedRemoteBugs(
remote_old_ids, oldest_lastchecked))
else:
# Don't ask the remote system about more than
# batch_size bugs at once, but keep asking until we
# run out of bugs to ask about or we have batch_size
# bugs to check.
remote_old_ids_to_check = []
for index in xrange(0, len(remote_old_ids), batch_size):
remote_old_ids_to_check.extend(
remotesystem.getModifiedRemoteBugs(
remote_old_ids[index : index + batch_size],
oldest_lastchecked))
if len(remote_old_ids_to_check) >= batch_size:
break
else:
remote_old_ids_to_check = remote_old_ids
# We'll create our remote_ids_to_check list so that it's
# prioritized. We include remote IDs in priority order:
# 1. IDs with comments.
# 2. IDs that haven't been checked.
# 3. Everything else.
remote_ids_to_check = chain(
remote_ids_with_comments, remote_new_ids, remote_old_ids_to_check)
if batch_size != BATCH_SIZE_UNLIMITED:
# Some remote bug IDs may appear in more than one list so
# we must filter the list before slicing.
remote_ids_to_check = islice(
unique(remote_ids_to_check), batch_size)
# Stuff the IDs in a set.
remote_ids_to_check = set(remote_ids_to_check)
# Make sure that unmodified_remote_ids only includes IDs that
# could have been checked but which weren't modified on the
# remote server and which haven't been listed for checking
# otherwise (i.e. because they have comments to be pushed).
unmodified_remote_ids = set(remote_old_ids)
unmodified_remote_ids.difference_update(remote_old_ids_to_check)
unmodified_remote_ids.difference_update(remote_ids_to_check)
all_remote_ids = remote_ids_to_check.union(unmodified_remote_ids)
return {
'remote_ids_to_check': sorted(remote_ids_to_check),
'all_remote_ids': sorted(all_remote_ids),
'unmodified_remote_ids': sorted(unmodified_remote_ids),
}
def _getExternalBugTrackersAndWatches(self, bug_tracker, bug_watches):
"""Return an `ExternalBugTracker` instance for `bug_tracker`."""
with self.transaction:
num_watches = bug_tracker.watches.count()
remotesystem = (
externalbugtracker.get_external_bugtracker(bug_tracker))
# We special-case the Gnome Bugzilla.
is_gnome_bugzilla = bug_tracker == (
getUtility(ILaunchpadCelebrities).gnome_bugzilla)
# Probe the remote system for additional capabilities.
remotesystem_to_use = remotesystem.getExternalBugTrackerToUse()
# Try to hint at how many bug watches to check each time.
suggest_batch_size(remotesystem_to_use, num_watches)
if (is_gnome_bugzilla and remotesystem_to_use.sync_comments):
# If there are no products to sync comments for, disable
# comment sync and return.
if len(self._syncable_gnome_products) == 0:
remotesystem_to_use.sync_comments = False
return [
(remotesystem_to_use, bug_watches),
]
syncable_watches = []
other_watches = []
with self.transaction:
reload(bug_watches)
remote_bug_ids = [
bug_watch.remotebug for bug_watch in bug_watches]
remote_products = (
remotesystem_to_use.getProductsForRemoteBugs(
remote_bug_ids))
with self.transaction:
reload(bug_watches)
for bug_watch in bug_watches:
if (remote_products.get(bug_watch.remotebug) in
self._syncable_gnome_products):
syncable_watches.append(bug_watch)
else:
other_watches.append(bug_watch)
# For bug watches on remote bugs that are against products
# in the _syncable_gnome_products list - i.e. ones with which
# we want to sync comments - we return a BugzillaAPI
# instance with sync_comments=True, otherwise we return a
# similar BugzillaAPI instance, but with sync_comments=False.
remotesystem_for_syncables = remotesystem_to_use
remotesystem_for_others = copy(remotesystem_to_use)
remotesystem_for_others.sync_comments = False
return [
(remotesystem_for_syncables, syncable_watches),
(remotesystem_for_others, other_watches),
]
else:
return [
(remotesystem_to_use, bug_watches),
]