class AptCacheTests(TestCase): """ Tests for :class:`pts.core.utils.packages.AptCache`. """ @staticmethod def stub_acquire(source_records, dest_dir, debian_dir_only, content): # Create a file in the destination directory file_name = 'temp' file_path = os.path.join(dest_dir, file_name) # Create a file of the given size with open(file_path, 'wb') as f: f.write(content) return None, 'ekrem' def create_cache(self): """ Helper method which creates an :class:`pts.core.utils.packages.AptCache` instance which is used for testing. Some of its methods are replaced by mocks and stubs to avoid HTTP calls. """ self.cache = AptCache() self.cache._get_apt_source_records = mock.MagicMock() self.cache._get_format = mock.MagicMock(return_value='1.0') self.cache._extract_dpkg_source = mock.MagicMock() self.cached_files = [] self.cache._get_all_cached_files = mock.MagicMock( return_value=self.cached_files) self.cache._match_index_file_to_repository = mock.MagicMock() def set_stub_acquire_content(self, content): """ Helper method which sets the content of a file which is created by the cache instance when retrieve_source is called. """ self.cache._apt_acquire_package = mock.MagicMock(side_effect=curry( AptCacheTests.stub_acquire, content=content)) def set_stub_cached_files_for_repository(self, repository, files): """ Helper method adds the given list of files to the stub list of cached files for a given repository. :param repository: The repository to which these files are associated. :type repository: :class:`Repository <pts.core.models.Repository>` :param files: List of cached file names. The function uses the list to build the stub by prefixing the names with expected repository identifiers. """ # Build the prefix from the repository's URI and suite base_uri = repository.uri.rstrip('/') if base_uri.startswith('http://'): base_uri = base_uri[7:] prefix = base_uri + '/' + repository.suite + '/' prefix = prefix.replace('/', '_') for file_name in files: self.cached_files.append(prefix + file_name) self.cache._match_index_file_to_repository.return_value = repository def assert_cache_size_equal(self, size): self.assertEqual(size, self.cache.cache_size) def test_cache_size_increase_after_acquire(self): """ Tests that the cache correctly increases its size after acquiring new files. """ with make_temp_directory('-pts-cache') as cache_directory: with self.settings( PTS_CACHE_DIRECTORY=cache_directory, PTS_APT_CACHE_MAX_SIZE=10): self.create_cache() # Sanity check: old size is 0 as nothing was ever cached in the # brand new directory self.assert_cache_size_equal(0) content = b'a' * 5 # 5 bytes self.set_stub_acquire_content(content) self.cache.retrieve_source('dummy-package', '1.0.0') self.assert_cache_size_equal(5) def test_cache_multiple_insert_no_remove(self): """ Tests that the cache does not remove packages unless the size limit is exceeded. """ with make_temp_directory('-pts-cache') as cache_directory: with self.settings( PTS_CACHE_DIRECTORY=cache_directory, PTS_APT_CACHE_MAX_SIZE=10): self.create_cache() # Sanity check: old size is 0 as nothing was ever cached in the # brand new directory self.assert_cache_size_equal(0) content = b'a' * 5 # 5 bytes self.set_stub_acquire_content(content) # Add one file. self.cache.retrieve_source('dummy-package', '1.0.0') self.assert_cache_size_equal(5) # Same content in another file self.set_stub_acquire_content(content) self.cache.retrieve_source('package', '1.0.0') # Both files are now saved. self.assert_cache_size_equal(10) def test_clear_cache(self): """ Tests that the cache removes packages when it exceeds its allocated size. """ with make_temp_directory('-pts-cache') as cache_directory: with self.settings( PTS_CACHE_DIRECTORY=cache_directory, PTS_APT_CACHE_MAX_SIZE=10): self.create_cache() # Sanity check: old size is 0 as nothing was ever cached in the # brand new directory self.assert_cache_size_equal(0) initial_content = b'a' * 11 self.set_stub_acquire_content(initial_content) # Set initial source content self.cache.retrieve_source('dummy-package', '1.0.0') self.assert_cache_size_equal(11) content = b'a' * 7 self.set_stub_acquire_content(content) self.cache.retrieve_source('package', '1.0.0') # Only the second content is found in the package self.assert_cache_size_equal(7) def test_get_sources_for_repository(self): """ Tests that the cache correctly returns a list of cached Sources files for a given repository. """ with make_temp_directory('-pts-cache') as cache_directory: with self.settings(PTS_CACHE_DIRECTORY=cache_directory): self.create_cache() repository = Repository.objects.create( name='stable', shorthand='stable', uri='http://cdn.debian.net/debian/dists', suite='stable') expected_source_files = [ 'main_source_Sources', 'contrib_source_Sources', ] files = expected_source_files + [ 'Release', 'main_binary-amd64_Packages', ] self.set_stub_cached_files_for_repository(repository, files) sources = self.cache.get_sources_files_for_repository(repository) self.assertEqual(len(expected_source_files), len(sources)) for expected_source, returned_source in zip( expected_source_files, sources): self.assertTrue(returned_source.endswith(expected_source)) def test_get_packages_for_repository(self): """ Tests that the cache correctly returns a list of cached Packages files for a given repository. """ with make_temp_directory('-pts-cache') as cache_directory: with self.settings(PTS_CACHE_DIRECTORY=cache_directory): self.create_cache() repository = Repository.objects.create( name='stable', shorthand='stable', uri='http://cdn.debian.net/debian/dists', suite='stable') expected_packages_files = [ 'main_binary-amd64_Packages', 'main_binary-i386_Packages', ] files = expected_packages_files + [ 'Release', 'main_source_Sources', ] self.set_stub_cached_files_for_repository(repository, files) packages = self.cache.get_packages_files_for_repository(repository) self.assertEqual(len(expected_packages_files), len(packages)) for expected, returned in zip( expected_packages_files, packages): self.assertTrue(returned.endswith(expected))
class ExtractSourcePackageFiles(BaseTask): """ A task which extracts some files from a new source package version. The extracted files are: - debian/changelog - debian/copyright - debian/rules - debian/control - debian/watch """ DEPENDS_ON_EVENTS = ("new-source-package-version",) PRODUCES_EVENTS = ("source-files-extracted",) ALL_FILES_TO_EXTRACT = ("changelog", "copyright", "rules", "control", "watch") def __init__(self, *args, **kwargs): super(ExtractSourcePackageFiles, self).__init__(*args, **kwargs) self.cache = None def extract_files(self, source_package, files_to_extract=None): """ Extract files for just the given source package. :type source_package: :class:`SourcePackage <pts.core.models.SourcePackage>` :type files_to_extract: An iterable of file names which should be extracted """ if self.cache is None: self.cache = AptCache() source_directory = self.cache.retrieve_source( source_package.source_package_name.name, source_package.version, debian_directory_only=True ) debian_directory = os.path.join(source_directory, "debian") if files_to_extract is None: files_to_extract = self.ALL_FILES_TO_EXTRACT for file_name in files_to_extract: file_path = os.path.join(debian_directory, file_name) if not os.path.exists(file_path): continue with open(file_path, "r") as f: extracted_file = File(f) ExtractedSourceFile.objects.create( source_package=source_package, extracted_file=extracted_file, name=file_name ) def _execute_initial(self): """ When the task is directly ran, instead of relying on events to know which packages' source files should be retrieved, the task scans all existing packages and adds any missing source packages for each of them. """ # First remove all source files which are no longer to be included. qs = ExtractedSourceFile.objects.exclude(name__in=self.ALL_FILES_TO_EXTRACT) qs.delete() # Retrieves the packages and all the associated files with each of them # in only two db queries. source_packages = SourcePackage.objects.all() source_packages.prefetch_related("extracted_source_files") # Find the difference of packages and extract only those for each # package for source_package in source_packages: extracted_files = [extracted_file.name for extracted_file in source_package.extracted_source_files.all()] files_to_extract = [ file_name for file_name in self.ALL_FILES_TO_EXTRACT if file_name not in extracted_files ] if files_to_extract: try: self.extract_files(source_package, files_to_extract) except: logger.exception( "Problem extracting source files for" " {pkg} version {ver}".format(pkg=source_package, ver=source_package.version) ) def execute(self): if self.is_initial_task(): return self._execute_initial() # When the task is not the initial task, then all the packages it # should process should come from received events. new_version_pks = [event.arguments["pk"] for event in self.get_all_events()] source_packages = SourcePackage.objects.filter(pk__in=new_version_pks) source_packages = source_packages.select_related() for source_package in source_packages: try: self.extract_files(source_package) except: logger.exception( "Problem extracting source files for" " {pkg} version {ver}".format(pkg=source_package, ver=source_package.version) ) self.raise_event("source-files-extracted")