class AptCacheTests(TestCase):
"""
Tests for :class:`pts.core.utils.packages.AptCache`.
"""
@staticmethod
def stub_acquire(source_records, dest_dir, debian_dir_only, content):
# Create a file in the destination directory
file_name = 'temp'
file_path = os.path.join(dest_dir, file_name)
# Create a file of the given size
with open(file_path, 'wb') as f:
f.write(content)
return None, 'ekrem'
def create_cache(self):
"""
Helper method which creates an :class:`pts.core.utils.packages.AptCache`
instance which is used for testing. Some of its methods are replaced by
mocks and stubs to avoid HTTP calls.
"""
self.cache = AptCache()
self.cache._get_apt_source_records = mock.MagicMock()
self.cache._get_format = mock.MagicMock(return_value='1.0')
self.cache._extract_dpkg_source = mock.MagicMock()
self.cached_files = []
self.cache._get_all_cached_files = mock.MagicMock(
return_value=self.cached_files)
self.cache._match_index_file_to_repository = mock.MagicMock()
def set_stub_acquire_content(self, content):
"""
Helper method which sets the content of a file which is created by the
cache instance when retrieve_source is called.
"""
self.cache._apt_acquire_package = mock.MagicMock(side_effect=curry(
AptCacheTests.stub_acquire, content=content))
def set_stub_cached_files_for_repository(self, repository, files):
"""
Helper method adds the given list of files to the stub list of cached
files for a given repository.
:param repository: The repository to which these files are associated.
:type repository: :class:`Repository <pts.core.models.Repository>`
:param files: List of cached file names. The function uses the list to
build the stub by prefixing the names with expected repository
identifiers.
"""
# Build the prefix from the repository's URI and suite
base_uri = repository.uri.rstrip('/')
if base_uri.startswith('http://'):
base_uri = base_uri[7:]
prefix = base_uri + '/' + repository.suite + '/'
prefix = prefix.replace('/', '_')
for file_name in files:
self.cached_files.append(prefix + file_name)
self.cache._match_index_file_to_repository.return_value = repository
def assert_cache_size_equal(self, size):
self.assertEqual(size, self.cache.cache_size)
def test_cache_size_increase_after_acquire(self):
"""
Tests that the cache correctly increases its size after acquiring new
files.
"""
with make_temp_directory('-pts-cache') as cache_directory:
with self.settings(
PTS_CACHE_DIRECTORY=cache_directory,
PTS_APT_CACHE_MAX_SIZE=10):
self.create_cache()
# Sanity check: old size is 0 as nothing was ever cached in the
# brand new directory
self.assert_cache_size_equal(0)
content = b'a' * 5 # 5 bytes
self.set_stub_acquire_content(content)
self.cache.retrieve_source('dummy-package', '1.0.0')
self.assert_cache_size_equal(5)
def test_cache_multiple_insert_no_remove(self):
"""
Tests that the cache does not remove packages unless the size limit is
exceeded.
"""
with make_temp_directory('-pts-cache') as cache_directory:
with self.settings(
PTS_CACHE_DIRECTORY=cache_directory,
PTS_APT_CACHE_MAX_SIZE=10):
self.create_cache()
# Sanity check: old size is 0 as nothing was ever cached in the
# brand new directory
self.assert_cache_size_equal(0)
content = b'a' * 5 # 5 bytes
self.set_stub_acquire_content(content)
# Add one file.
self.cache.retrieve_source('dummy-package', '1.0.0')
self.assert_cache_size_equal(5)
# Same content in another file
self.set_stub_acquire_content(content)
self.cache.retrieve_source('package', '1.0.0')
# Both files are now saved.
self.assert_cache_size_equal(10)
def test_clear_cache(self):
"""
Tests that the cache removes packages when it exceeds its allocated
size.
"""
with make_temp_directory('-pts-cache') as cache_directory:
with self.settings(
PTS_CACHE_DIRECTORY=cache_directory,
PTS_APT_CACHE_MAX_SIZE=10):
self.create_cache()
# Sanity check: old size is 0 as nothing was ever cached in the
# brand new directory
self.assert_cache_size_equal(0)
initial_content = b'a' * 11
self.set_stub_acquire_content(initial_content)
# Set initial source content
self.cache.retrieve_source('dummy-package', '1.0.0')
self.assert_cache_size_equal(11)
content = b'a' * 7
self.set_stub_acquire_content(content)
self.cache.retrieve_source('package', '1.0.0')
# Only the second content is found in the package
self.assert_cache_size_equal(7)
def test_get_sources_for_repository(self):
"""
Tests that the cache correctly returns a list of cached Sources files
for a given repository.
"""
with make_temp_directory('-pts-cache') as cache_directory:
with self.settings(PTS_CACHE_DIRECTORY=cache_directory):
self.create_cache()
repository = Repository.objects.create(
name='stable',
shorthand='stable',
uri='http://cdn.debian.net/debian/dists',
suite='stable')
expected_source_files = [
'main_source_Sources',
'contrib_source_Sources',
]
files = expected_source_files + [
'Release',
'main_binary-amd64_Packages',
]
self.set_stub_cached_files_for_repository(repository, files)
sources = self.cache.get_sources_files_for_repository(repository)
self.assertEqual(len(expected_source_files), len(sources))
for expected_source, returned_source in zip(
expected_source_files, sources):
self.assertTrue(returned_source.endswith(expected_source))
def test_get_packages_for_repository(self):
"""
Tests that the cache correctly returns a list of cached Packages files
for a given repository.
"""
with make_temp_directory('-pts-cache') as cache_directory:
with self.settings(PTS_CACHE_DIRECTORY=cache_directory):
self.create_cache()
repository = Repository.objects.create(
name='stable',
shorthand='stable',
uri='http://cdn.debian.net/debian/dists',
suite='stable')
expected_packages_files = [
'main_binary-amd64_Packages',
'main_binary-i386_Packages',
]
files = expected_packages_files + [
'Release',
'main_source_Sources',
]
self.set_stub_cached_files_for_repository(repository, files)
packages = self.cache.get_packages_files_for_repository(repository)
self.assertEqual(len(expected_packages_files), len(packages))
for expected, returned in zip(
expected_packages_files, packages):
self.assertTrue(returned.endswith(expected))
class ExtractSourcePackageFiles(BaseTask):
"""
A task which extracts some files from a new source package version.
The extracted files are:
- debian/changelog
- debian/copyright
- debian/rules
- debian/control
- debian/watch
"""
DEPENDS_ON_EVENTS = ("new-source-package-version",)
PRODUCES_EVENTS = ("source-files-extracted",)
ALL_FILES_TO_EXTRACT = ("changelog", "copyright", "rules", "control", "watch")
def __init__(self, *args, **kwargs):
super(ExtractSourcePackageFiles, self).__init__(*args, **kwargs)
self.cache = None
def extract_files(self, source_package, files_to_extract=None):
"""
Extract files for just the given source package.
:type source_package: :class:`SourcePackage <pts.core.models.SourcePackage>`
:type files_to_extract: An iterable of file names which should be
extracted
"""
if self.cache is None:
self.cache = AptCache()
source_directory = self.cache.retrieve_source(
source_package.source_package_name.name, source_package.version, debian_directory_only=True
)
debian_directory = os.path.join(source_directory, "debian")
if files_to_extract is None:
files_to_extract = self.ALL_FILES_TO_EXTRACT
for file_name in files_to_extract:
file_path = os.path.join(debian_directory, file_name)
if not os.path.exists(file_path):
continue
with open(file_path, "r") as f:
extracted_file = File(f)
ExtractedSourceFile.objects.create(
source_package=source_package, extracted_file=extracted_file, name=file_name
)
def _execute_initial(self):
"""
When the task is directly ran, instead of relying on events to know
which packages' source files should be retrieved, the task scans all
existing packages and adds any missing source packages for each of
them.
"""
# First remove all source files which are no longer to be included.
qs = ExtractedSourceFile.objects.exclude(name__in=self.ALL_FILES_TO_EXTRACT)
qs.delete()
# Retrieves the packages and all the associated files with each of them
# in only two db queries.
source_packages = SourcePackage.objects.all()
source_packages.prefetch_related("extracted_source_files")
# Find the difference of packages and extract only those for each
# package
for source_package in source_packages:
extracted_files = [extracted_file.name for extracted_file in source_package.extracted_source_files.all()]
files_to_extract = [
file_name for file_name in self.ALL_FILES_TO_EXTRACT if file_name not in extracted_files
]
if files_to_extract:
try:
self.extract_files(source_package, files_to_extract)
except:
logger.exception(
"Problem extracting source files for"
" {pkg} version {ver}".format(pkg=source_package, ver=source_package.version)
)
def execute(self):
if self.is_initial_task():
return self._execute_initial()
# When the task is not the initial task, then all the packages it
# should process should come from received events.
new_version_pks = [event.arguments["pk"] for event in self.get_all_events()]
source_packages = SourcePackage.objects.filter(pk__in=new_version_pks)
source_packages = source_packages.select_related()
for source_package in source_packages:
try:
self.extract_files(source_package)
except:
logger.exception(
"Problem extracting source files for"
" {pkg} version {ver}".format(pkg=source_package, ver=source_package.version)
)
self.raise_event("source-files-extracted")
