def test_source_overrides_latest_only_is_returned(self):
# When the spn is published multiple times in the given
# archive/distroseries, the latest publication's overrides are
# returned.
spn = self.factory.makeSourcePackageName()
distroseries = self.factory.makeDistroSeries()
published_spr = self.factory.makeSourcePackageRelease(
sourcepackagename=spn)
self.factory.makeSourcePackagePublishingHistory(
sourcepackagerelease=published_spr,
distroseries=distroseries,
status=PackagePublishingStatus.PUBLISHED)
spr = self.factory.makeSourcePackageRelease(sourcepackagename=spn)
spph = self.factory.makeSourcePackagePublishingHistory(
sourcepackagerelease=spr, distroseries=distroseries)
overrides = FromExistingOverridePolicy(
spph.distroseries.main_archive, spph.distroseries,
spph.pocket).calculateSourceOverrides({spn: SourceOverride()})
self.assertEqual(
{
spn:
SourceOverride(component=spph.component,
section=spph.section,
version=spph.sourcepackagerelease.version,
new=False)
}, overrides)
def test_fallback_sources(self):
# The fallback policy runs through a sequence of policies until
# all overrides are fulfilled.
universe = getUtility(IComponentSet)['universe']
spns = [self.factory.makeSourcePackageName()]
expected = {spns[0]: SourceOverride(component=universe, new=True)}
distroseries = self.factory.makeDistroSeries()
pocket = self.factory.getAnyPocket()
for i in range(8):
spph = self.factory.makeSourcePackagePublishingHistory(
distroseries=distroseries,
archive=distroseries.main_archive,
pocket=pocket)
spns.append(spph.sourcepackagerelease.sourcepackagename)
expected[spph.sourcepackagerelease.sourcepackagename] = (
SourceOverride(component=spph.component,
section=spph.section,
version=spph.sourcepackagerelease.version,
new=False))
spns.append(self.factory.makeSourcePackageName())
expected[spns[-1]] = SourceOverride(component=universe, new=True)
policy = FallbackOverridePolicy([
FromExistingOverridePolicy(distroseries.main_archive, distroseries,
pocket),
UnknownOverridePolicy(distroseries.main_archive, distroseries,
pocket)
])
overrides = policy.calculateSourceOverrides(
dict((spn, SourceOverride()) for spn in spns))
self.assertEqual(10, len(overrides))
self.assertEqual(expected, overrides)
def test_ubuntu_override_policy_sources(self):
# The Ubuntu policy incorporates both the existing and the unknown
# policy.
universe = getUtility(IComponentSet)['universe']
spns = [self.factory.makeSourcePackageName()]
expected = [SourceOverride(spns[0], universe, None)]
distroseries = self.factory.makeDistroSeries()
pocket = self.factory.getAnyPocket()
for i in xrange(8):
spph = self.factory.makeSourcePackagePublishingHistory(
distroseries=distroseries, archive=distroseries.main_archive,
pocket=pocket)
spns.append(spph.sourcepackagerelease.sourcepackagename)
expected.append(
SourceOverride(
spph.sourcepackagerelease.sourcepackagename,
spph.component, spph.section))
spns.append(self.factory.makeSourcePackageName())
expected.append(SourceOverride(spns[-1], universe, None))
policy = UbuntuOverridePolicy()
overrides = policy.calculateSourceOverrides(
distroseries.main_archive, distroseries, pocket, spns)
self.assertEqual(10, len(overrides))
sorted_expected = sorted(
expected, key=attrgetter("source_package_name.name"))
sorted_overrides = sorted(
overrides, key=attrgetter("source_package_name.name"))
self.assertEqual(sorted_expected, sorted_overrides)
def test_source_overrides_can_include_deleted(self):
# include_deleted=True causes Deleted publications to be
# considered too.
spn = self.factory.makeSourcePackageName()
distroseries = self.factory.makeDistroSeries()
spr = self.factory.makeSourcePackageRelease(sourcepackagename=spn)
spph = self.factory.makeSourcePackagePublishingHistory(
archive=distroseries.main_archive,
distroseries=distroseries,
sourcepackagerelease=spr,
status=PackagePublishingStatus.PUBLISHED)
deleted_spr = self.factory.makeSourcePackageRelease(
sourcepackagename=spn)
deleted_spph = self.factory.makeSourcePackagePublishingHistory(
archive=distroseries.main_archive,
distroseries=distroseries,
sourcepackagerelease=deleted_spr,
status=PackagePublishingStatus.DELETED,
pocket=spph.pocket)
# With include_deleted=False only the Published ancestry is
# found.
overrides = FromExistingOverridePolicy(
distroseries.main_archive, distroseries,
spph.pocket).calculateSourceOverrides({spn: SourceOverride(spn)})
self.assertEqual(
{
spn:
SourceOverride(component=spph.component,
section=spph.section,
version=spph.sourcepackagerelease.version,
new=False)
}, overrides)
# But with include_deleted=True the newer Deleted publication is
# used.
overrides = FromExistingOverridePolicy(
distroseries.main_archive,
distroseries,
spph.pocket,
include_deleted=True).calculateSourceOverrides(
{spn: SourceOverride(spn)})
self.assertEqual(
{
spn:
SourceOverride(
component=deleted_spph.component,
section=deleted_spph.section,
version=deleted_spph.sourcepackagerelease.version,
new=True)
}, overrides)
def test_no_sources(self):
# Source overrides are never returned.
self.assertEqual({},
FromSourceOverridePolicy().calculateSourceOverrides({
self.factory.makeSourcePackageName():
SourceOverride()
}))
def test_source_overrides(self):
# When the spn is published in the given archive/distroseries, the
# overrides for that archive/distroseries are returned.
spph = self.factory.makeSourcePackagePublishingHistory()
policy = FromExistingOverridePolicy(spph.distroseries.main_archive,
spph.distroseries, spph.pocket)
overrides = policy.calculateSourceOverrides(
{spph.sourcepackagerelease.sourcepackagename: SourceOverride()})
expected = {
spph.sourcepackagerelease.sourcepackagename:
SourceOverride(component=spph.component,
section=spph.section,
version=spph.sourcepackagerelease.version,
new=False)
}
self.assertEqual(expected, overrides)
def test_sources(self):
policy = ConstantOverridePolicy(component=self.factory.makeComponent(),
section=self.factory.makeSection(),
phased_update_percentage=50,
new=True)
spn = self.factory.makeSourcePackageName()
self.assertEqual(
{
spn:
SourceOverride(component=policy.component,
section=policy.section,
new=True)
},
policy.calculateSourceOverrides({
spn:
SourceOverride(component=self.factory.makeComponent(),
section=self.factory.makeSection(),
new=False)
}))
def check_copy_permissions(person, archive, series, pocket, sources):
"""Check that `person` has permission to copy a package.
:param person: User attempting the upload.
:param archive: Destination `Archive`.
:param series: Destination `DistroSeries`.
:param pocket: Destination `Pocket`.
:param sources: Sequence of `SourcePackagePublishingHistory`s for the
packages to be copied.
:raises CannotCopy: If the copy is not allowed.
"""
# Circular import.
from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
if person is None:
raise CannotCopy("Cannot check copy permissions (no requester).")
if len(sources) > 1:
# Bulk-load the data we'll need from each source publication.
load_related(SourcePackageRelease, sources, ["sourcepackagereleaseID"])
# If there is a requester, check that they have upload permission into
# the destination (archive, component, pocket). This check is done
# here rather than in the security adapter because it requires more
# info than is available in the security adapter.
for source in sources:
dest_series = series or source.distroseries
spn = source.sourcepackagerelease.sourcepackagename
policy = archive.getOverridePolicy(dest_series, pocket)
override = policy.calculateSourceOverrides(
{spn: SourceOverride(component=source.component)})[spn]
# Is the destination pocket open at all?
reason = archive.checkUploadToPocket(dest_series,
pocket,
person=person)
if reason is not None:
raise CannotCopy(reason)
# If the package exists in the target, make sure the person has
# upload permission for its component. Otherwise, any upload
# permission on this archive will do.
reason = archive.verifyUpload(person,
spn,
override.component,
dest_series,
strict_component=(override.new == False),
pocket=pocket)
if reason is not None:
# Queue admins are allowed to copy even if they can't upload.
if not archive.canAdministerQueue(person, override.component,
pocket, dest_series):
raise CannotCopy(reason)
def test_unknown_sources(self):
# The unknown policy uses a default component based on the
# pre-override component.
for component in ('contrib', 'non-free'):
self.factory.makeComponent(component)
distroseries = self.factory.makeDistroSeries()
spns = [self.factory.makeSourcePackageName() for i in range(3)]
policy = UnknownOverridePolicy(distroseries.main_archive, distroseries,
PackagePublishingPocket.RELEASE)
overrides = policy.calculateSourceOverrides(
dict((spn,
SourceOverride(
component=getUtility(IComponentSet)[component]))
for spn, component in zip(spns, ('main', 'contrib',
'non-free'))))
expected = dict(
(spn,
SourceOverride(component=getUtility(IComponentSet)[component],
new=True))
for spn, component in zip(spns, ('universe', 'multiverse',
'multiverse')))
self.assertEqual(expected, overrides)
def getSourceOverride(self):
"""Fetch an `ISourceOverride` from the metadata."""
component_name = self.component_name
section_name = self.section_name
try:
component = getUtility(IComponentSet)[component_name]
except NotFoundError:
component = None
try:
section = getUtility(ISectionSet)[section_name]
except NotFoundError:
section = None
return SourceOverride(component=component, section=section)
def test_unknown_sources(self):
# If the unknown policy is used, it does no checks, just returns the
# defaults.
spph = self.factory.makeSourcePackagePublishingHistory()
policy = UnknownOverridePolicy()
overrides = policy.calculateSourceOverrides(
spph.distroseries.main_archive, spph.distroseries, spph.pocket,
(spph.sourcepackagerelease.sourcepackagename,))
universe = getUtility(IComponentSet)['universe']
expected = [
SourceOverride(
spph.sourcepackagerelease.sourcepackagename, universe,
None)]
self.assertEqual(expected, overrides)
def test_source_overrides_pocket(self):
# If the spn is not published in the given pocket, no changes
# are made.
spn = self.factory.makeSourcePackageName()
distroseries = self.factory.makeDistroSeries()
self.factory.makeSourcePackagePublishingHistory(
archive=distroseries.main_archive,
distroseries=distroseries,
pocket=PackagePublishingPocket.RELEASE,
sourcepackagename=spn)
overrides = FromExistingOverridePolicy(
distroseries.main_archive, distroseries,
PackagePublishingPocket.PROPOSED).calculateSourceOverrides(
{spn: SourceOverride()})
self.assertEqual(0, len(overrides))
overrides = FromExistingOverridePolicy(
distroseries.main_archive, distroseries,
PackagePublishingPocket.RELEASE).calculateSourceOverrides(
{spn: SourceOverride()})
self.assertEqual(1, len(overrides))
overrides = FromExistingOverridePolicy(
distroseries.main_archive, distroseries,
None).calculateSourceOverrides({spn: SourceOverride()})
self.assertEqual(1, len(overrides))
def getSourceOverride(self):
"""Fetch an `ISourceOverride` from the metadata."""
name = self.package_name
component_name = self.component_name
section_name = self.section_name
source_package_name = getUtility(ISourcePackageNameSet)[name]
try:
component = getUtility(IComponentSet)[component_name]
except NotFoundError:
component = None
try:
section = getUtility(ISectionSet)[section_name]
except NotFoundError:
section = None
return SourceOverride(source_package_name, component, section)
def test_binaries(self):
# Binaries are overridden with the component from their
# corresponding source override, if one was provided.
bpn = self.factory.makeBinaryPackageName()
other_bpn = self.factory.makeBinaryPackageName()
component = self.factory.makeComponent()
random_component = self.factory.makeComponent()
self.assertEqual(
{(bpn, None): BinaryOverride(component=component, new=True)},
FromSourceOverridePolicy().calculateBinaryOverrides({
(bpn, None):
BinaryOverride(
component=random_component,
source_override=SourceOverride(component=component)),
(other_bpn, None):
BinaryOverride(component=random_component)
}))
def test_source_overrides_constant_query_count(self):
# The query count is constant, no matter how many sources are
# checked.
spns = []
distroseries = self.factory.makeDistroSeries()
pocket = self.factory.getAnyPocket()
for i in range(10):
spph = self.factory.makeSourcePackagePublishingHistory(
distroseries=distroseries,
archive=distroseries.main_archive,
pocket=pocket)
spns.append(spph.sourcepackagerelease.sourcepackagename)
flush_database_caches()
distroseries.main_archive
bulk.reload(spns)
policy = FromExistingOverridePolicy(spph.distroseries.main_archive,
spph.distroseries, spph.pocket)
with StormStatementRecorder() as recorder:
policy.calculateSourceOverrides(
dict((spn, SourceOverride()) for spn in spns))
self.assertThat(recorder, HasQueryCount(Equals(3)))
def _checkPolicies(self,
source_name,
source_component=None,
auto_approve=False):
# This helper will only return if it's safe to carry on with the
# copy, otherwise it raises SuspendJobException to tell the job
# runner to suspend the job.
override_policy = self.target_archive.getOverridePolicy(
self.target_distroseries, self.target_pocket)
overrides = override_policy.calculateSourceOverrides(
{source_name: SourceOverride(component=source_component)})
override = overrides[source_name]
copy_policy = self.getPolicyImplementation()
# Put the (existing or default) override in the metadata.
self.addSourceOverride(override)
auto_approve = auto_approve and self.requester_can_admin_target
if override.new:
approve_new = auto_approve or copy_policy.autoApproveNew(
self.target_archive, self.target_distroseries,
self.target_pocket)
if not approve_new:
# There's no existing package with the same name and the
# policy says unapproved, so we poke it in the NEW queue.
self._createPackageUpload()
raise SuspendJobException
else:
# The package is not new (it has ancestry) so check the copy
# policy for existing packages.
approve_existing = auto_approve or copy_policy.autoApprove(
self.target_archive, self.target_distroseries,
self.target_pocket)
if not approve_existing:
self._createPackageUpload(unapproved=True)
raise SuspendJobException
def _do_direct_copy(source,
archive,
series,
pocket,
include_binaries,
override=None,
close_bugs=True,
create_dsd_job=True,
close_bugs_since_version=None,
creator=None,
sponsor=None,
packageupload=None,
phased_update_percentage=None,
logger=None):
"""Copy publishing records to another location.
Copy each item of the given list of `SourcePackagePublishingHistory`
to the given destination if they are not yet available (previously
copied).
Also copy published binaries for each source if requested to. Again,
only copy binaries that were not yet copied before.
:param source: an `ISourcePackagePublishingHistory`.
:param archive: the target `IArchive`.
:param series: the target `IDistroSeries`, if None is given the same
current source distroseries will be used as destination.
:param pocket: the target `PackagePublishingPocket`.
:param include_binaries: optional boolean, controls whether or
not the published binaries for each given source should be also
copied along with the source.
:param override: An `IOverride` as per do_copy().
:param close_bugs: A boolean indicating whether or not bugs on the
copied publication should be closed.
:param create_dsd_job: A boolean indicating whether or not a dsd job
should be created for the new source publication.
:param close_bugs_since_version: If close_bugs is True,
then this parameter says which changelog entries to parse looking
for bugs to close. See `close_bugs_for_sourcepackagerelease`.
:param creator: the requester `IPerson`.
:param sponsor: the sponsor `IPerson`, if this copy is being sponsored.
:param packageupload: The `IPackageUpload` that caused this publication
to be created.
:param phased_update_percentage: The phased update percentage to apply
to the copied publication.
:param logger: An optional logger.
:return: a list of `ISourcePackagePublishingHistory` and
`BinaryPackagePublishingHistory` corresponding to the copied
publications.
"""
copies = []
custom_files = []
# Copy source if it's not yet copied.
source_in_destination = archive.getPublishedSources(
name=source.sourcepackagerelease.name,
exact_match=True,
version=source.sourcepackagerelease.version,
status=active_publishing_status,
distroseries=series,
pocket=pocket)
policy = archive.getOverridePolicy(
series, pocket, phased_update_percentage=phased_update_percentage)
if override is None and policy is not None:
# Only one override can be returned so take the first
# element of the returned list.
override = policy.calculateSourceOverrides(
{source.sourcepackagerelease.sourcepackagename:
SourceOverride()})[source.sourcepackagerelease.sourcepackagename]
if source_in_destination.is_empty():
# If no manual overrides were specified and the archive has an
# override policy then use that policy to get overrides.
source_copy = source.copyTo(series,
pocket,
archive,
override,
create_dsd_job=create_dsd_job,
creator=creator,
sponsor=sponsor,
packageupload=packageupload)
if close_bugs:
close_bugs_for_sourcepublication(source_copy,
close_bugs_since_version)
copies.append(source_copy)
else:
source_copy = source_in_destination.first()
if source_copy.packageupload is not None:
custom_files.extend(source_copy.packageupload.customfiles)
if include_binaries:
# Copy missing binaries for the matching architectures in the
# destination series. ISPPH.getBuiltBinaries() return only unique
# publication per binary package releases (i.e. excludes irrelevant
# arch-indep publications) and IBPPH.copy is prepared to expand
# arch-indep publications.
binary_copies = getUtility(IPublishingSet).copyBinaries(
archive,
series,
pocket,
source.getBuiltBinaries(),
policy=policy,
source_override=override)
if binary_copies is not None:
copies.extend(binary_copies)
binary_uploads = set(bpph.binarypackagerelease.build.package_upload
for bpph in binary_copies)
for binary_upload in binary_uploads:
if binary_upload is not None:
custom_files.extend(binary_upload.customfiles)
if custom_files:
# Custom uploads aren't modelled as publication history records, so
# we have to send these through the upload queue.
custom_copier = CustomUploadsCopier(series,
target_pocket=pocket,
target_archive=archive)
for custom in custom_files:
if custom_copier.isCopyable(custom):
custom_copier.copyUpload(custom)
# Always ensure the needed builds exist in the copy destination
# after copying the binaries.
# XXX cjwatson 2012-06-22 bug=869308: Fails to honour P-a-s.
source_copy.createMissingBuilds(logger=logger)
return copies