def determine_file_class_and_name(filename): """Determine the name and PackageUploadFile subclass for the filename.""" source_match = re_issource.match(filename) binary_match = re_isadeb.match(filename) buildinfo_match = re_isbuildinfo.match(filename) if source_match: package = source_match.group(1) if (determine_source_file_type(filename) == SourcePackageFileType.DSC): cls = DSCFile else: cls = SourceUploadFile elif binary_match: package = binary_match.group(1) cls = { BinaryPackageFileType.DEB: DebBinaryUploadFile, BinaryPackageFileType.DDEB: DdebBinaryUploadFile, BinaryPackageFileType.UDEB: UdebBinaryUploadFile, }[determine_binary_file_type(filename)] elif buildinfo_match: package = buildinfo_match.group(1) cls = BuildInfoFile else: raise CannotDetermineFileTypeError( "Could not determine the type of %r" % filename) return package, cls
def addFile(self, file, filetype=None): """See ISourcePackageRelease.""" if filetype is None: filetype = determine_source_file_type(file.filename) sprf = SourcePackageReleaseFile(sourcepackagerelease=self, filetype=filetype, libraryfile=file) del get_property_cache(self).files return sprf
def _getFileByName(self, filename): """Return the corresponding file reference in the policy context. If the filename ends in '.orig.tar.gz', then we look for it in the distribution primary archive as well, with the PPA file taking precedence in case it's found in both archives. This is needed so that PPA uploaders don't have to waste bandwidth uploading huge upstream tarballs that are already published in the target distribution. When the file reference is found, its corresponding LibraryFileAlias and Archive are returned. :param filename: string containing the exact name of the wanted file. :return: a tuple containing a `ILibraryFileAlias` corresponding to the matching file and an `Archive` where it was published. :raise: `NotFoundError` when the wanted file could not be found. """ # We cannot check the archive purpose for partner archives here, # because the archive override rules have not been applied yet. # Uploads destined for the Ubuntu main archive and the 'partner' # component will eventually end up in the partner archive though. if (self.policy.archive.purpose == ArchivePurpose.PRIMARY and self.component_name == 'partner'): archives = [ getUtility(IArchiveSet).getByDistroPurpose( distribution=self.policy.distro, purpose=ArchivePurpose.PARTNER) ] elif (self.policy.archive.purpose == ArchivePurpose.PPA and determine_source_file_type(filename) in (SourcePackageFileType.ORIG_TARBALL, SourcePackageFileType.COMPONENT_ORIG_TARBALL)): archives = [self.policy.archive, self.policy.distro.main_archive] else: archives = [self.policy.archive] archives = [archive for archive in archives if archive is not None] library_file = None for archive in archives: try: library_file = archive.getFileByName(filename) self.logger.debug("%s found in %s" % (filename, archive.displayname)) return library_file, archive except NotFoundError: pass raise NotFoundError(filename)
def _getFileByName(self, filename): """Return the corresponding file reference in the policy context. If the filename ends in '.orig.tar.gz', then we look for it in the distribution primary archive as well, with the PPA file taking precedence in case it's found in both archives. This is needed so that PPA uploaders don't have to waste bandwidth uploading huge upstream tarballs that are already published in the target distribution. When the file reference is found, its corresponding LibraryFileAlias and Archive are returned. :param filename: string containing the exact name of the wanted file. :return: a tuple containing a `ILibraryFileAlias` corresponding to the matching file and an `Archive` where it was published. :raise: `NotFoundError` when the wanted file could not be found. """ # We cannot check the archive purpose for partner archives here, # because the archive override rules have not been applied yet. # Uploads destined for the Ubuntu main archive and the 'partner' # component will eventually end up in the partner archive though. if (self.policy.archive.purpose == ArchivePurpose.PRIMARY and self.component_name == 'partner'): archives = [ getUtility(IArchiveSet).getByDistroPurpose( distribution=self.policy.distro, purpose=ArchivePurpose.PARTNER)] elif (self.policy.archive.purpose == ArchivePurpose.PPA and determine_source_file_type(filename) in ( SourcePackageFileType.ORIG_TARBALL, SourcePackageFileType.COMPONENT_ORIG_TARBALL)): archives = [self.policy.archive, self.policy.distro.main_archive] else: archives = [self.policy.archive] archives = [archive for archive in archives if archive is not None] library_file = None for archive in archives: try: library_file = archive.getFileByName(filename) self.logger.debug( "%s found in %s" % (filename, archive.displayname)) return library_file, archive except NotFoundError: pass raise NotFoundError(filename)
def _getFileByName(self, filename): """Return the corresponding file reference in the policy context. If the filename ends in '.orig.tar.gz', then we look for it in the distribution primary archive as well, with the PPA file taking precedence in case it's found in both archives. This is needed so that PPA uploaders don't have to waste bandwidth uploading huge upstream tarballs that are already published in the target distribution. When the file reference is found, its corresponding LibraryFileAlias and Archive are returned. :param filename: string containing the exact name of the wanted file. :return: a tuple containing a `ILibraryFileAlias` corresponding to the matching file and an `Archive` where it was published. :raise: `NotFoundError` when the wanted file could not be found. """ if (self.policy.archive.purpose == ArchivePurpose.PPA and determine_source_file_type(filename) in ( SourcePackageFileType.ORIG_TARBALL, SourcePackageFileType.COMPONENT_ORIG_TARBALL, SourcePackageFileType.ORIG_TARBALL_SIGNATURE, SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE, )): archives = [self.policy.archive, self.policy.distro.main_archive] else: archives = [self.policy.archive] archives = [archive for archive in archives if archive is not None] library_file = None for archive in archives: try: library_file = archive.getFileByName(filename) self.logger.debug("%s found in %s" % (filename, archive.displayname)) return library_file, archive except NotFoundError: pass raise NotFoundError(filename)
def _check_sourceful_consistency(self): """Heuristic checks on a sourceful upload. Raises AssertionError when called for a non-sourceful upload. Ensures a sourceful upload has exactly one DSC. All further source checks are performed later by the DSC. """ assert self.sourceful, ( "Source consistency check called for a non-source upload") dsc = len([ file for file in self.changes.files if determine_source_file_type( file.filename) == SourcePackageFileType.DSC ]) # It is never sane to upload more than one source at a time. if dsc > 1: self.reject("Changes file lists more than one .dsc") if dsc == 0: self.reject("Sourceful upload without a .dsc")
def _check_sourceful_consistency(self): """Heuristic checks on a sourceful upload. Raises AssertionError when called for a non-sourceful upload. Ensures a sourceful upload has exactly one DSC. All further source checks are performed later by the DSC. """ assert self.sourceful, ( "Source consistency check called for a non-source upload") dsc = len([ file for file in self.changes.files if determine_source_file_type(file.filename) == SourcePackageFileType.DSC]) # It is never sane to upload more than one source at a time. if dsc > 1: self.reject("Changes file lists more than one .dsc") if dsc == 0: self.reject("Sourceful upload without a .dsc")
def determine_file_class_and_name(filename): """Determine the name and PackageUploadFile subclass for the filename.""" source_match = re_issource.match(filename) binary_match = re_isadeb.match(filename) if source_match: package = source_match.group(1) if determine_source_file_type(filename) == SourcePackageFileType.DSC: cls = DSCFile else: cls = SourceUploadFile elif binary_match: package = binary_match.group(1) cls = { BinaryPackageFileType.DEB: DebBinaryUploadFile, BinaryPackageFileType.DDEB: DdebBinaryUploadFile, BinaryPackageFileType.UDEB: UdebBinaryUploadFile, }[determine_binary_file_type(filename)] else: raise CannotDetermineFileTypeError("Could not determine the type of %r" % filename) return package, cls
def checkFiles(self): """Check if mentioned files are present and match. We don't use the NascentUploadFile.verify here, only verify size and checksum. """ file_type_counts = { SourcePackageFileType.DIFF: 0, SourcePackageFileType.ORIG_TARBALL: 0, SourcePackageFileType.ORIG_TARBALL_SIGNATURE: 0, SourcePackageFileType.DEBIAN_TARBALL: 0, SourcePackageFileType.NATIVE_TARBALL: 0, } component_orig_tar_counts = {} component_orig_tar_signature_counts = {} bzip2_count = 0 xz_count = 0 files_missing = False for sub_dsc_file in self.files: file_type = determine_source_file_type(sub_dsc_file.filename) if file_type is None: yield UploadError('Unknown file: ' + sub_dsc_file.filename) continue if file_type == SourcePackageFileType.COMPONENT_ORIG_TARBALL: # Split the count by component name. component = re_is_component_orig_tar_ext.match( get_source_file_extension(sub_dsc_file.filename)).group(1) if component not in component_orig_tar_counts: component_orig_tar_counts[component] = 0 component_orig_tar_counts[component] += 1 elif (file_type == SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE): # Split the count by component name. component = re_is_component_orig_tar_ext_sig.match( get_source_file_extension(sub_dsc_file.filename)).group(1) if component not in component_orig_tar_signature_counts: component_orig_tar_signature_counts[component] = 0 component_orig_tar_signature_counts[component] += 1 else: file_type_counts[file_type] += 1 if sub_dsc_file.filename.endswith('.bz2'): bzip2_count += 1 elif sub_dsc_file.filename.endswith('.xz'): xz_count += 1 try: library_file, file_archive = self._getFileByName( sub_dsc_file.filename) except NotFoundError as error: library_file = None file_archive = None else: # try to check dsc-mentioned file against its copy already # in librarian, if it's new (aka not found in librarian) # dismiss. It prevents us from having scary duplicated # filenames in Librarian and misapplied files in archive, # fixes bug # 38636 and friends. if sub_dsc_file.checksums['MD5'] != library_file.content.md5: yield UploadError( "File %s already exists in %s, but uploaded version " "has different contents. See more information about " "this error in " "https://help.launchpad.net/Packaging/UploadErrors." % (sub_dsc_file.filename, file_archive.displayname)) files_missing = True continue if not sub_dsc_file.exists_on_disk: if library_file is None: # Raises an error if the mentioned DSC file isn't # included in the upload neither published in the # context Distribution. yield UploadError( "Unable to find %s in upload or distribution." % (sub_dsc_file.filename)) files_missing = True continue # Pump the file through. self.logger.debug("Pumping %s out of the librarian" % (sub_dsc_file.filename)) library_file.open() target_file = open(sub_dsc_file.filepath, "wb") copy_and_close(library_file, target_file) for error in sub_dsc_file.verify(): yield error files_missing = True try: file_checker = format_to_file_checker_map[self.format] except KeyError: raise AssertionError("No file checker for source format %s." % self.format) for error in file_checker(self.filename, file_type_counts, component_orig_tar_counts, component_orig_tar_signature_counts, bzip2_count, xz_count): yield error if files_missing: yield UploadError("Files specified in DSC are broken or missing, " "skipping package unpack verification.") else: for error in self.unpackAndCheckSource(): # Pass on errors found when unpacking the source. yield error
def test_determine_source_file_type(self): """lp.archiveuploader.utils.determine_source_file_type should work.""" # .dsc -> DSC self.assertEqual( determine_source_file_type('foo_1.0-1.dsc'), SourcePackageFileType.DSC) # .diff.gz -> DIFF self.assertEqual( determine_source_file_type('foo_1.0-1.diff.gz'), SourcePackageFileType.DIFF) # DIFFs can only be gzipped. self.assertEqual( determine_source_file_type('foo_1.0.diff.bz2'), None) # Plain original tarballs can be gzipped or bzip2ed. self.assertEqual( determine_source_file_type('foo_1.0.orig.tar.gz'), SourcePackageFileType.ORIG_TARBALL) self.assertEqual( determine_source_file_type('foo_1.0.orig.tar.bz2'), SourcePackageFileType.ORIG_TARBALL) self.assertEqual( determine_source_file_type('foo_1.0.orig.tar.xz'), SourcePackageFileType.ORIG_TARBALL) # Component original tarballs too. self.assertEqual( determine_source_file_type('foo_1.0.orig-foo.tar.gz'), SourcePackageFileType.COMPONENT_ORIG_TARBALL) self.assertEqual( determine_source_file_type('foo_1.0.orig-bar.tar.bz2'), SourcePackageFileType.COMPONENT_ORIG_TARBALL) self.assertEqual( determine_source_file_type('foo_1.0.orig-bar.tar.xz'), SourcePackageFileType.COMPONENT_ORIG_TARBALL) # And Debian tarballs... self.assertEqual( determine_source_file_type('foo_1.0-1.debian.tar.gz'), SourcePackageFileType.DEBIAN_TARBALL) self.assertEqual( determine_source_file_type('foo_1.0-2.debian.tar.bz2'), SourcePackageFileType.DEBIAN_TARBALL) self.assertEqual( determine_source_file_type('foo_1.0-2.debian.tar.xz'), SourcePackageFileType.DEBIAN_TARBALL) # And even native tarballs! self.assertEqual( determine_source_file_type('foo_1.0.tar.gz'), SourcePackageFileType.NATIVE_TARBALL) self.assertEqual( determine_source_file_type('foo_1.0.tar.bz2'), SourcePackageFileType.NATIVE_TARBALL) self.assertEqual( determine_source_file_type('foo_1.0.tar.xz'), SourcePackageFileType.NATIVE_TARBALL) # (Component) original tarball signatures are detected for any # supported compression method. self.assertEqual( determine_source_file_type('foo_1.0.orig.tar.gz.asc'), SourcePackageFileType.ORIG_TARBALL_SIGNATURE) self.assertEqual( determine_source_file_type('foo_1.0.orig.tar.bz2.asc'), SourcePackageFileType.ORIG_TARBALL_SIGNATURE) self.assertEqual( determine_source_file_type('foo_1.0.orig.tar.xz.asc'), SourcePackageFileType.ORIG_TARBALL_SIGNATURE) self.assertEqual( determine_source_file_type('foo_1.0.orig-foo.tar.gz.asc'), SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE) self.assertEqual( determine_source_file_type('foo_1.0.orig-bar.tar.bz2.asc'), SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE) self.assertEqual( determine_source_file_type('foo_1.0.orig-bar.tar.xz.asc'), SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE) self.assertIsNone(determine_source_file_type('foo_1.0')) self.assertIsNone(determine_source_file_type('foo_1.0.blah.gz'))
def filetype(self): return determine_source_file_type(self.filename)
def addFile(self, file): """See ISourcePackageRelease.""" return SourcePackageReleaseFile(sourcepackagerelease=self, filetype=determine_source_file_type( file.filename), libraryfile=file)
def addFile(self, file): """See ISourcePackageRelease.""" return SourcePackageReleaseFile( sourcepackagerelease=self, filetype=determine_source_file_type(file.filename), libraryfile=file)
def test_determine_source_file_type(self): """lp.archiveuploader.utils.determine_source_file_type should work.""" # .dsc -> DSC self.assertEquals( determine_source_file_type('foo_1.0-1.dsc'), SourcePackageFileType.DSC) # .diff.gz -> DIFF self.assertEquals( determine_source_file_type('foo_1.0-1.diff.gz'), SourcePackageFileType.DIFF) # DIFFs can only be gzipped. self.assertEquals( determine_source_file_type('foo_1.0.diff.bz2'), None) # Plain original tarballs can be gzipped or bzip2ed. self.assertEquals( determine_source_file_type('foo_1.0.orig.tar.gz'), SourcePackageFileType.ORIG_TARBALL) self.assertEquals( determine_source_file_type('foo_1.0.orig.tar.bz2'), SourcePackageFileType.ORIG_TARBALL) self.assertEquals( determine_source_file_type('foo_1.0.orig.tar.xz'), SourcePackageFileType.ORIG_TARBALL) # Component original tarballs too. self.assertEquals( determine_source_file_type('foo_1.0.orig-foo.tar.gz'), SourcePackageFileType.COMPONENT_ORIG_TARBALL) self.assertEquals( determine_source_file_type('foo_1.0.orig-bar.tar.bz2'), SourcePackageFileType.COMPONENT_ORIG_TARBALL) self.assertEquals( determine_source_file_type('foo_1.0.orig-bar.tar.xz'), SourcePackageFileType.COMPONENT_ORIG_TARBALL) # And Debian tarballs... self.assertEquals( determine_source_file_type('foo_1.0-1.debian.tar.gz'), SourcePackageFileType.DEBIAN_TARBALL) self.assertEquals( determine_source_file_type('foo_1.0-2.debian.tar.bz2'), SourcePackageFileType.DEBIAN_TARBALL) self.assertEquals( determine_source_file_type('foo_1.0-2.debian.tar.xz'), SourcePackageFileType.DEBIAN_TARBALL) # And even native tarballs! self.assertEquals( determine_source_file_type('foo_1.0.tar.gz'), SourcePackageFileType.NATIVE_TARBALL) self.assertEquals( determine_source_file_type('foo_1.0.tar.bz2'), SourcePackageFileType.NATIVE_TARBALL) self.assertEquals( determine_source_file_type('foo_1.0.tar.xz'), SourcePackageFileType.NATIVE_TARBALL) self.assertEquals(None, determine_source_file_type('foo_1.0')) self.assertEquals(None, determine_source_file_type('foo_1.0.blah.gz'))
def checkFiles(self): """Check if mentioned files are present and match. We don't use the NascentUploadFile.verify here, only verify size and checksum. """ file_type_counts = { SourcePackageFileType.DIFF: 0, SourcePackageFileType.ORIG_TARBALL: 0, SourcePackageFileType.DEBIAN_TARBALL: 0, SourcePackageFileType.NATIVE_TARBALL: 0, } component_orig_tar_counts = {} bzip2_count = 0 xz_count = 0 files_missing = False for sub_dsc_file in self.files: file_type = determine_source_file_type(sub_dsc_file.filename) if file_type is None: yield UploadError('Unknown file: ' + sub_dsc_file.filename) continue if file_type == SourcePackageFileType.COMPONENT_ORIG_TARBALL: # Split the count by component name. component = re_is_component_orig_tar_ext.match( get_source_file_extension(sub_dsc_file.filename)).group(1) if component not in component_orig_tar_counts: component_orig_tar_counts[component] = 0 component_orig_tar_counts[component] += 1 else: file_type_counts[file_type] += 1 if sub_dsc_file.filename.endswith('.bz2'): bzip2_count += 1 elif sub_dsc_file.filename.endswith('.xz'): xz_count += 1 try: library_file, file_archive = self._getFileByName( sub_dsc_file.filename) except NotFoundError as error: library_file = None file_archive = None else: # try to check dsc-mentioned file against its copy already # in librarian, if it's new (aka not found in librarian) # dismiss. It prevents us from having scary duplicated # filenames in Librarian and misapplied files in archive, # fixes bug # 38636 and friends. if sub_dsc_file.checksums['MD5'] != library_file.content.md5: yield UploadError( "File %s already exists in %s, but uploaded version " "has different contents. See more information about " "this error in " "https://help.launchpad.net/Packaging/UploadErrors." % (sub_dsc_file.filename, file_archive.displayname)) files_missing = True continue if not sub_dsc_file.exists_on_disk: if library_file is None: # Raises an error if the mentioned DSC file isn't # included in the upload neither published in the # context Distribution. yield UploadError( "Unable to find %s in upload or distribution." % (sub_dsc_file.filename)) files_missing = True continue # Pump the file through. self.logger.debug("Pumping %s out of the librarian" % ( sub_dsc_file.filename)) library_file.open() target_file = open(sub_dsc_file.filepath, "wb") copy_and_close(library_file, target_file) for error in sub_dsc_file.verify(): yield error files_missing = True try: file_checker = format_to_file_checker_map[self.format] except KeyError: raise AssertionError( "No file checker for source format %s." % self.format) for error in file_checker( self.filename, file_type_counts, component_orig_tar_counts, bzip2_count, xz_count): yield error if files_missing: yield UploadError( "Files specified in DSC are broken or missing, " "skipping package unpack verification.") else: for error in self.unpackAndCheckSource(): # Pass on errors found when unpacking the source. yield error
def createSourcePackageRelease(self, src, distroseries): """Create a SourcePackagerelease and db dependencies if needed. Returns the created SourcePackageRelease, or None if it failed. """ displayname, emailaddress = src.maintainer comment = 'when the %s package was imported into %s' % ( src.package, distroseries.displayname) maintainer = getUtility(IPersonSet).ensurePerson( emailaddress, displayname, PersonCreationRationale.SOURCEPACKAGEIMPORT, comment=comment) # XXX Debonzi 2005-05-16: Check it later. # if src.dsc_signing_key_owner: # key = self.getGPGKey(src.dsc_signing_key, # *src.dsc_signing_key_owner) # else: key = None to_upload = check_not_in_librarian(src.files, src.archive_root, src.directory) # Create the SourcePackageRelease (SPR) componentID = self.distro_handler.getComponentByName(src.component).id sectionID = self.distro_handler.ensureSection(src.section).id maintainer_line = "%s <%s>" % (displayname, emailaddress) name = self.ensureSourcePackageName(src.package) spr = SourcePackageRelease( section=sectionID, creator=maintainer.id, component=componentID, sourcepackagename=name.id, maintainer=maintainer.id, dscsigningkey=key, urgency=ChangesFile.urgency_map[src.urgency], dateuploaded=src.date_uploaded, dsc=src.dsc, copyright=src.copyright, version=src.version, changelog_entry=src.changelog_entry, builddepends=src.build_depends, builddependsindep=src.build_depends_indep, build_conflicts=src.build_conflicts, build_conflicts_indep=src.build_conflicts_indep, architecturehintlist=src.architecture, format=SourcePackageType.DPKG, upload_distroseries=distroseries.id, dsc_format=src.format, dsc_maintainer_rfc822=maintainer_line, dsc_standards_version=src.standards_version, dsc_binaries=", ".join(src.binaries), upload_archive=distroseries.main_archive) log.info('Source Package Release %s (%s) created' % (name.name, src.version)) # Upload the changelog to the Librarian if src.changelog is not None: changelog_lfa = getUtility(ILibraryFileAliasSet).create( "changelog", len(src.changelog), StringIO(src.changelog), "text/x-debian-source-changelog") spr.changelog = changelog_lfa # Insert file into the library and create the # SourcePackageReleaseFile entry on lp db. for fname, path in to_upload: alias = getLibraryAlias(path, fname) SourcePackageReleaseFile( sourcepackagerelease=spr.id, libraryfile=alias, filetype=determine_source_file_type(fname)) log.info('Package file %s included into library' % fname) return spr