def got_file(ignored, filename, out_file, out_file_name): try: # If the requested file is the 'buildlog' compress it # using gzip before storing in Librarian. if file_sha1 == 'buildlog': out_file = open(out_file_name) filename += '.gz' out_file_name += '.gz' gz_file = gzip.GzipFile(out_file_name, mode='wb') copy_and_close(out_file, gz_file) os.remove(out_file_name.replace('.gz', '')) # Reopen the file, seek to its end position, count and seek # to beginning, ready for adding to the Librarian. out_file = open(out_file_name) out_file.seek(0, 2) bytes_written = out_file.tell() out_file.seek(0) library_file = getUtility(ILibraryFileAliasSet).create( filename, bytes_written, out_file, contentType=filenameToContentType(filename), restricted=private) finally: # Remove the temporary file. getFile() closes the file # object. os.remove(out_file_name) return library_file.id
def download_file(destination_path, libraryfile): """Download a file from the librarian to the destination path. :param destination_path: Absolute destination path (where the file should be downloaded to). :type destination_path: ``str`` :param libraryfile: The librarian file that is to be downloaded. :type libraryfile: ``LibraryFileAlias`` """ libraryfile.open() destination_file = open(destination_path, 'w') copy_and_close(libraryfile, destination_file)
def _compressFile(self, path): """Compress the file in the given path using bzip2. The compressed file will be in the same path and old file will be removed. :return: the path to the compressed file. """ bz2_path = '%s.bz2' % path copy_and_close(open(path), bz2.BZ2File(bz2_path, mode='w')) os.remove(path) return bz2_path
def publish(cls, packageupload, libraryfilealias, logger=None): """See `ICustomUploadHandler`.""" temp_dir = tempfile.mkdtemp() try: tarfile_path = os.path.join(temp_dir, libraryfilealias.filename) temp_file = open(tarfile_path, "wb") libraryfilealias.open() copy_and_close(libraryfilealias, temp_file) suite = packageupload.distroseries.getSuite(packageupload.pocket) upload = cls(logger=logger) upload.process(packageupload.archive, tarfile_path, suite) finally: shutil.rmtree(temp_dir)
def process(self, packageupload, libraryfilealias): pubconf = getPubConfig(packageupload.archive) if pubconf.metaroot is None: if self.logger is not None: self.logger.debug( "Skipping meta-data for archive without metaroot.") return dest_file = os.path.join(pubconf.metaroot, libraryfilealias.filename) if not os.path.isdir(pubconf.metaroot): os.makedirs(pubconf.metaroot, 0o755) # At this point we now have a directory of the format: # <person_name>/meta/<ppa_name> # We're ready to copy the file out of the librarian into it. file_obj = open(dest_file, "wb") libraryfilealias.open() copy_and_close(libraryfilealias, file_obj)
def addFile(self, component, sha1, contents): """See DiskPool.addFile.""" assert component in HARDCODED_COMPONENT_ORDER targetpath = self.pathFor(component) if not os.path.exists(os.path.dirname(targetpath)): os.makedirs(os.path.dirname(targetpath)) if self.file_component: # There's something on disk. Check hash. if sha1 != self.file_hash: raise PoolFileOverwriteError('%s != %s for %s' % (sha1, self.file_hash, self.pathFor(self.file_component))) if (component == self.file_component or component in self.symlink_components): # The file is already here return FileAddActionEnum.NONE else: # The file is present in a different component, # make a symlink. relative_symlink( self.pathFor(self.file_component), targetpath) self.symlink_components.add(component) # Then fix to ensure the right component is linked. self._sanitiseLinks() return FileAddActionEnum.SYMLINK_ADDED # If we get to here, we want to write the file. assert not os.path.exists(targetpath) self.debug("Making new file in %s for %s/%s" % (component, self.source, self.filename)) file_to_write = _diskpool_atomicfile( targetpath, "wb", rootpath=self.temppath) contents.open() copy_and_close(contents, file_to_write) self.file_component = component return FileAddActionEnum.FILE_ADDED
def checkFiles(self): """Check if mentioned files are present and match. We don't use the NascentUploadFile.verify here, only verify size and checksum. """ file_type_counts = { SourcePackageFileType.DIFF: 0, SourcePackageFileType.ORIG_TARBALL: 0, SourcePackageFileType.ORIG_TARBALL_SIGNATURE: 0, SourcePackageFileType.DEBIAN_TARBALL: 0, SourcePackageFileType.NATIVE_TARBALL: 0, } component_orig_tar_counts = {} component_orig_tar_signature_counts = {} bzip2_count = 0 xz_count = 0 files_missing = False for sub_dsc_file in self.files: file_type = determine_source_file_type(sub_dsc_file.filename) if file_type is None: yield UploadError('Unknown file: ' + sub_dsc_file.filename) continue if file_type == SourcePackageFileType.COMPONENT_ORIG_TARBALL: # Split the count by component name. component = re_is_component_orig_tar_ext.match( get_source_file_extension(sub_dsc_file.filename)).group(1) if component not in component_orig_tar_counts: component_orig_tar_counts[component] = 0 component_orig_tar_counts[component] += 1 elif (file_type == SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE): # Split the count by component name. component = re_is_component_orig_tar_ext_sig.match( get_source_file_extension(sub_dsc_file.filename)).group(1) if component not in component_orig_tar_signature_counts: component_orig_tar_signature_counts[component] = 0 component_orig_tar_signature_counts[component] += 1 else: file_type_counts[file_type] += 1 if sub_dsc_file.filename.endswith('.bz2'): bzip2_count += 1 elif sub_dsc_file.filename.endswith('.xz'): xz_count += 1 try: library_file, file_archive = self._getFileByName( sub_dsc_file.filename) except NotFoundError as error: library_file = None file_archive = None else: # try to check dsc-mentioned file against its copy already # in librarian, if it's new (aka not found in librarian) # dismiss. It prevents us from having scary duplicated # filenames in Librarian and misapplied files in archive, # fixes bug # 38636 and friends. if sub_dsc_file.checksums['MD5'] != library_file.content.md5: yield UploadError( "File %s already exists in %s, but uploaded version " "has different contents. See more information about " "this error in " "https://help.launchpad.net/Packaging/UploadErrors." % (sub_dsc_file.filename, file_archive.displayname)) files_missing = True continue if not sub_dsc_file.exists_on_disk: if library_file is None: # Raises an error if the mentioned DSC file isn't # included in the upload neither published in the # context Distribution. yield UploadError( "Unable to find %s in upload or distribution." % (sub_dsc_file.filename)) files_missing = True continue # Pump the file through. self.logger.debug("Pumping %s out of the librarian" % (sub_dsc_file.filename)) library_file.open() target_file = open(sub_dsc_file.filepath, "wb") copy_and_close(library_file, target_file) for error in sub_dsc_file.verify(): yield error files_missing = True try: file_checker = format_to_file_checker_map[self.format] except KeyError: raise AssertionError("No file checker for source format %s." % self.format) for error in file_checker(self.filename, file_type_counts, component_orig_tar_counts, component_orig_tar_signature_counts, bzip2_count, xz_count): yield error if files_missing: yield UploadError("Files specified in DSC are broken or missing, " "skipping package unpack verification.") else: for error in self.unpackAndCheckSource(): # Pass on errors found when unpacking the source. yield error
def fake_getFile(sum, file): dummy_tar = os.path.join(os.path.dirname(__file__), 'dummy_templates.tar.gz') tar_file = open(dummy_tar) copy_and_close(tar_file, file) return defer.succeed(None)
def performDiff(self): """See `IPackageDiff`. This involves creating a temporary directory, downloading the files from both SPRs involved from the librarian, running debdiff, storing the output in the librarian and updating the PackageDiff record. """ # Make sure the files associated with the two source packages are # still available in the librarian. if self._countDeletedLFAs() > 0: self.status = PackageDiffStatus.FAILED return blacklist = config.packagediff.blacklist.split() if self.from_source.sourcepackagename.name in blacklist: self.status = PackageDiffStatus.FAILED return # Create the temporary directory where the files will be # downloaded to and where the debdiff will be performed. tmp_dir = tempfile.mkdtemp() try: directions = ('from', 'to') # Keep track of the files belonging to the respective packages. downloaded = dict(zip(directions, ([], []))) # Make it easy to iterate over packages. packages = dict(zip(directions, (self.from_source, self.to_source))) # Iterate over the packages to be diff'ed. for direction, package in packages.iteritems(): # Create distinct directory locations for # 'from' and 'to' files. absolute_path = os.path.join(tmp_dir, direction) os.makedirs(absolute_path) # Download the files associated with each package in # their corresponding relative location. for file in package.files: the_name = file.libraryfile.filename relative_location = os.path.join(direction, the_name) downloaded[direction].append(relative_location) destination_path = os.path.join(absolute_path, the_name) download_file(destination_path, file.libraryfile) # All downloads are done. Construct the name of the resulting # diff file. result_filename = '%s_%s_%s.diff' % ( self.from_source.sourcepackagename.name, self.from_source.version, self.to_source.version) # Perform the actual diff operation. return_code, stderr = perform_deb_diff(tmp_dir, result_filename, downloaded['from'], downloaded['to']) # `debdiff` failed, mark the package diff request accordingly # and return. 0 means no differences, 1 means they differ. # Note that pre-Karmic debdiff will return 0 even if they differ. if return_code not in (0, 1): self.status = PackageDiffStatus.FAILED return # Compress the generated diff. out_file = open(os.path.join(tmp_dir, result_filename)) gzip_result_filename = result_filename + '.gz' gzip_file_path = os.path.join(tmp_dir, gzip_result_filename) gzip_file = gzip.GzipFile(gzip_file_path, mode='wb') copy_and_close(out_file, gzip_file) # Calculate the compressed size. gzip_size = os.path.getsize(gzip_file_path) # Upload the compressed diff to librarian and update # the package diff request. gzip_file = open(gzip_file_path) try: librarian_set = getUtility(ILibraryFileAliasSet) self.diff_content = librarian_set.create( gzip_result_filename, gzip_size, gzip_file, 'application/gzipped-patch', restricted=self.private) finally: gzip_file.close() # Last but not least, mark the diff as COMPLETED. self.date_fulfilled = UTC_NOW self.status = PackageDiffStatus.COMPLETED finally: shutil.rmtree(tmp_dir)
def checkFiles(self): """Check if mentioned files are present and match. We don't use the NascentUploadFile.verify here, only verify size and checksum. """ file_type_counts = { SourcePackageFileType.DIFF: 0, SourcePackageFileType.ORIG_TARBALL: 0, SourcePackageFileType.DEBIAN_TARBALL: 0, SourcePackageFileType.NATIVE_TARBALL: 0, } component_orig_tar_counts = {} bzip2_count = 0 xz_count = 0 files_missing = False for sub_dsc_file in self.files: file_type = determine_source_file_type(sub_dsc_file.filename) if file_type is None: yield UploadError('Unknown file: ' + sub_dsc_file.filename) continue if file_type == SourcePackageFileType.COMPONENT_ORIG_TARBALL: # Split the count by component name. component = re_is_component_orig_tar_ext.match( get_source_file_extension(sub_dsc_file.filename)).group(1) if component not in component_orig_tar_counts: component_orig_tar_counts[component] = 0 component_orig_tar_counts[component] += 1 else: file_type_counts[file_type] += 1 if sub_dsc_file.filename.endswith('.bz2'): bzip2_count += 1 elif sub_dsc_file.filename.endswith('.xz'): xz_count += 1 try: library_file, file_archive = self._getFileByName( sub_dsc_file.filename) except NotFoundError as error: library_file = None file_archive = None else: # try to check dsc-mentioned file against its copy already # in librarian, if it's new (aka not found in librarian) # dismiss. It prevents us from having scary duplicated # filenames in Librarian and misapplied files in archive, # fixes bug # 38636 and friends. if sub_dsc_file.checksums['MD5'] != library_file.content.md5: yield UploadError( "File %s already exists in %s, but uploaded version " "has different contents. See more information about " "this error in " "https://help.launchpad.net/Packaging/UploadErrors." % (sub_dsc_file.filename, file_archive.displayname)) files_missing = True continue if not sub_dsc_file.exists_on_disk: if library_file is None: # Raises an error if the mentioned DSC file isn't # included in the upload neither published in the # context Distribution. yield UploadError( "Unable to find %s in upload or distribution." % (sub_dsc_file.filename)) files_missing = True continue # Pump the file through. self.logger.debug("Pumping %s out of the librarian" % ( sub_dsc_file.filename)) library_file.open() target_file = open(sub_dsc_file.filepath, "wb") copy_and_close(library_file, target_file) for error in sub_dsc_file.verify(): yield error files_missing = True try: file_checker = format_to_file_checker_map[self.format] except KeyError: raise AssertionError( "No file checker for source format %s." % self.format) for error in file_checker( self.filename, file_type_counts, component_orig_tar_counts, bzip2_count, xz_count): yield error if files_missing: yield UploadError( "Files specified in DSC are broken or missing, " "skipping package unpack verification.") else: for error in self.unpackAndCheckSource(): # Pass on errors found when unpacking the source. yield error
def performDiff(self): """See `IPackageDiff`. This involves creating a temporary directory, downloading the files from both SPRs involved from the librarian, running debdiff, storing the output in the librarian and updating the PackageDiff record. """ # Make sure the files associated with the two source packages are # still available in the librarian. if self._countDeletedLFAs() > 0: self.status = PackageDiffStatus.FAILED return # Create the temporary directory where the files will be # downloaded to and where the debdiff will be performed. tmp_dir = tempfile.mkdtemp() try: directions = ('from', 'to') # Keep track of the files belonging to the respective packages. downloaded = dict(zip(directions, ([], []))) # Make it easy to iterate over packages. packages = dict( zip(directions, (self.from_source, self.to_source))) # Iterate over the packages to be diff'ed. for direction, package in packages.iteritems(): # Create distinct directory locations for # 'from' and 'to' files. absolute_path = os.path.join(tmp_dir, direction) os.makedirs(absolute_path) # Download the files associated with each package in # their corresponding relative location. for file in package.files: the_name = file.libraryfile.filename relative_location = os.path.join(direction, the_name) downloaded[direction].append(relative_location) destination_path = os.path.join(absolute_path, the_name) download_file(destination_path, file.libraryfile) # All downloads are done. Construct the name of the resulting # diff file. result_filename = '%s_%s_%s.diff' % ( self.from_source.sourcepackagename.name, self.from_source.version, self.to_source.version) # Perform the actual diff operation. return_code, stderr = perform_deb_diff( tmp_dir, result_filename, downloaded['from'], downloaded['to']) # `debdiff` failed, mark the package diff request accordingly # and return. 0 means no differences, 1 means they differ. # Note that pre-Karmic debdiff will return 0 even if they differ. if return_code not in (0, 1): self.status = PackageDiffStatus.FAILED return # Compress the generated diff. out_file = open(os.path.join(tmp_dir, result_filename)) gzip_result_filename = result_filename + '.gz' gzip_file_path = os.path.join(tmp_dir, gzip_result_filename) gzip_file = gzip.GzipFile(gzip_file_path, mode='wb') copy_and_close(out_file, gzip_file) # Calculate the compressed size. gzip_size = os.path.getsize(gzip_file_path) # Upload the compressed diff to librarian and update # the package diff request. gzip_file = open(gzip_file_path) try: librarian_set = getUtility(ILibraryFileAliasSet) self.diff_content = librarian_set.create( gzip_result_filename, gzip_size, gzip_file, 'application/gzipped-patch', restricted=self.private) finally: gzip_file.close() # Last but not least, mark the diff as COMPLETED. self.date_fulfilled = UTC_NOW self.status = PackageDiffStatus.COMPLETED finally: shutil.rmtree(tmp_dir)
def fake_getFile(sum, file): dummy_tar = os.path.join( os.path.dirname(__file__), 'dummy_templates.tar.gz') tar_file = open(dummy_tar) copy_and_close(tar_file, file) return defer.succeed(None)