Esempio n. 1
0
 def setUp(self):
     self.job_state = mock.create_autospec(JobState)
     self.job_state.events_for_task.return_value = []
     self.job_state.processed_tasks = []
     self.job = mock.create_autospec(Job)
     self.job.job_state = self.job_state
     self.task = ExtractSourcePackageFiles()
     self.task.job = self.job
Esempio n. 2
0
 def setUp(self):
     self.task = ExtractSourcePackageFiles()
     self.srcpkg = self.create_source_package()
Esempio n. 3
0
class ExtractSourcePackageFilesTest(TestCase):
    """
    Tests for the task
    :class:`distro_tracker.extract_source_files.ExtractSourcePackageFiles`.
    """
    def setUp(self):
        self.task = ExtractSourcePackageFiles()
        self.srcpkg = self.create_source_package()

    def run_task(self, force_update=False):
        """
        Initiates the task run.

        :param bool force_update: force a full run
        """
        if force_update:
            self.task.initialize(force_update=True)
        self.task.execute()

    def setup_debian_dir(self, pkg_directory, files_to_create=None,
                         extra_files=[], contents=b'Contents'):
        debian_dir = os.path.join(pkg_directory, 'debian')
        os.makedirs(debian_dir)
        if files_to_create is None:
            files_to_create = self.task.ALL_FILES_TO_EXTRACT

        for file_name in itertools.chain(files_to_create, extra_files):
            file_path = os.path.join(debian_dir, file_name)
            with open(file_path, 'wb') as f:
                f.write(contents)

        return debian_dir

    def assertExtractedFilesInDB(self, expected=None):
        if expected is None:
            expected = self.task.ALL_FILES_TO_EXTRACT
        expected = set(expected)
        extracted = set([
            x.name for x in self.srcpkg.extracted_source_files.all()
        ])
        self.assertSetEqual(expected, extracted)

    def test_create_extracted_files(self, mock_cache):
        """
        Tests that the task creates an
        :class:`distro_tracker.core.models.ExtractedSourceFile` instance.
        """
        with make_temp_directory('dtracker-pkg-dir') as pkg_directory:
            mock_cache.return_value = pkg_directory
            self.setup_debian_dir(pkg_directory, ['changelog'])
            self.run_task()

        self.assertExtractedFilesInDB(['changelog'])

    def test_create_extracted_files_only_wanted_files(self, mock_cache):
        """
        Tests that the task creates an
        :class:`distro_tracker.core.models.ExtractedSourceFile` instance.
        """
        with make_temp_directory('dtracker-pkg-dir') as pkg_directory:
            mock_cache.return_value = pkg_directory
            self.setup_debian_dir(pkg_directory, extra_files=['other-file'])
            self.run_task()

        self.assertExtractedFilesInDB()

    def test_create_extracted_files_with_non_utf8_content(self, mock_cache):
        """
        Tests that the task creates an
        :class:`distro_tracker.core.models.ExtractedSourceFile` instance
        even when the content of the file is not valid UTF-8.
        """
        with make_temp_directory('dtracker-pkg-dir') as pkg_directory:
            mock_cache.return_value = pkg_directory
            self.setup_debian_dir(pkg_directory, ['changelog'],
                                  contents='Raphaël'.encode('latin1'))
            self.run_task()

        self.assertExtractedFilesInDB(['changelog'])

    def test_task_force_update_no_existing_files(self, mock_cache):
        """
        Test that the force_update mode extracts missing files for already
        processed source packages.
        """
        self.task.item_mark_processed(self.srcpkg)

        with make_temp_directory('dtracker-pkg-dir') as pkg_directory:
            mock_cache.return_value = pkg_directory
            self.setup_debian_dir(pkg_directory)
            self.run_task(force_update=True)

        self.assertExtractedFilesInDB()

    def test_task_with_existing_files(self, mock_cache):
        """
        Ensure task doesn't overwrite existing files when it re-process
        a package that was already processed.
        """
        # Make a previously extracted file.
        original_content = b'Original content'
        self.srcpkg.extracted_source_files.create(
            name='changelog',
            extracted_file=ContentFile(original_content, name='changelog'))

        with make_temp_directory('dtracker-pkg-dir') as pkg_directory:
            mock_cache.return_value = pkg_directory
            self.setup_debian_dir(pkg_directory)
            self.run_task()

        self.assertExtractedFilesInDB()

        # Check that the existing file was not changed.
        extracted_file = ExtractedSourceFile.objects.get(
            name='changelog', source_package=self.srcpkg)
        extracted_file.extracted_file.open()
        content = extracted_file.extracted_file.read()
        extracted_file.extracted_file.close()
        self.assertEqual(original_content, content)

    def test_task_remove_unwanted_file(self, mock_cache):
        # Make a previously extracted file that we no longer extract
        original_content = 'Original content'
        ExtractedSourceFile.objects.create(
            source_package=self.srcpkg,
            name='we-dont-want-this-any-more',
            extracted_file=ContentFile(original_content, name='changelog'))

        with make_temp_directory('dtracker-pkg-dir') as pkg_directory:
            mock_cache.return_value = pkg_directory
            self.setup_debian_dir(pkg_directory)
            self.run_task()

        self.assertExtractedFilesInDB()
Esempio n. 4
0
class ExtractSourcePackageFilesTest(TestCase):
    """
    Tests for the task
    :class:`distro_tracker.extract_source_files.ExtractSourcePackageFiles`.
    """
    def setUp(self):
        self.job_state = mock.create_autospec(JobState)
        self.job_state.events_for_task.return_value = []
        self.job_state.processed_tasks = []
        self.job = mock.create_autospec(Job)
        self.job.job_state = self.job_state
        self.task = ExtractSourcePackageFiles()
        self.task.job = self.job

    def add_mock_event(self, name, arguments):
        """
        Helper method adding mock events which the task will have access to
        when it runs.
        """
        self.job_state.events_for_task.return_value.append(
            Event(name=name, arguments=arguments)
        )

    def run_task(self, initial_task=False):
        """
        Initiates the task run.

        :param initial_task: An optional flag which if ``True`` means that the
            task should be ran as if it were directly passed to the
            :func:`distro_tracker.core.tasks.run_task` function.
        :type initial_task: Boolean
        """
        if initial_task:
            self.job_state.events_for_task.return_value = []
        else:
            # If it is not the initial task, add a dummy task to make it look
            # like that.
            self.job_state.processed_tasks = ['sometask']

        self.task.execute()

    @mock.patch('distro_tracker.extract_source_files.tracker_tasks.'
                'AptCache.retrieve_source')
    def test_create_extracted_files(self, mock_cache):
        """
        Tests that the task creates an
        :class:`distro_tracker.core.models.ExtractedSourceFile` instance.
        """
        name = SourcePackageName.objects.create(name='dummy-package')
        package = SourcePackage.objects.create(
            source_package_name=name, version='1.0.0')
        self.add_mock_event('new-source-package-version', {
            'pk': package.pk,
        })

        with make_temp_directory('dtracker-pkg-dir') as pkg_directory:
            debian_dir = os.path.join(pkg_directory, 'debian')
            os.makedirs(debian_dir)
            changelog_path = os.path.join(debian_dir, 'changelog')
            with open(changelog_path, 'w') as f:
                f.write('Contents')
            # This file should not be included in the extracted files.
            other_file = os.path.join(debian_dir, 'some-file')
            with open(other_file, 'w') as f:
                f.write('Contents')

            mock_cache.return_value = os.path.join(pkg_directory)

            self.run_task()

            # Check that the file was created.
            self.assertEqual(1, ExtractedSourceFile.objects.count())
            # Check that it has the correct name
            extracted_file = ExtractedSourceFile.objects.all()[0]
            self.assertEqual('changelog', extracted_file.name)

    @mock.patch('distro_tracker.extract_source_files.tracker_tasks.'
                'AptCache.retrieve_source')
    def test_create_extracted_files_only_wanted_files(self, mock_cache):
        """
        Tests that the task creates an
        :class:`distro_tracker.core.models.ExtractedSourceFile` instance.
        """
        name = SourcePackageName.objects.create(name='dummy-package')
        package = SourcePackage.objects.create(
            source_package_name=name, version='1.0.0')
        self.add_mock_event('new-source-package-version', {
            'pk': package.pk,
        })

        with make_temp_directory('dtracker-pkg-dir') as pkg_directory:
            debian_dir = os.path.join(pkg_directory, 'debian')
            os.makedirs(debian_dir)
            wanted_files = [
                'changelog',
                'copyright',
                'rules',
                'control',
                'watch',
            ]
            all_files = wanted_files + [
                'other-file',
            ]
            for file_name in all_files:
                file_path = os.path.join(debian_dir, file_name)
                with open(file_path, 'w') as f:
                    f.write('Contents')

            mock_cache.return_value = os.path.join(pkg_directory)

            self.run_task()

            # Check that only the wanted files are created!
            self.assertEqual(len(wanted_files),
                             ExtractedSourceFile.objects.count())
            extracted_names = [
                extracted_file.name
                for extracted_file in ExtractedSourceFile.objects.all()
            ]
            for wanted_file in wanted_files:
                self.assertIn(wanted_file, extracted_names)

    @mock.patch('distro_tracker.extract_source_files.tracker_tasks.'
                'AptCache.retrieve_source')
    def test_task_is_initial_no_existing_files(self, mock_cache):
        """
        Tests the task when it is run as the initial task, but there are no
        extracted files for existing packages.
        """
        name = SourcePackageName.objects.create(name='dummy-package')
        SourcePackage.objects.create(
            source_package_name=name, version='1.0.0')

        with make_temp_directory('dtracker-pkg-dir') as pkg_directory:
            debian_dir = os.path.join(pkg_directory, 'debian')
            os.makedirs(debian_dir)
            wanted_files = [
                'changelog',
                'copyright',
                'rules',
                'control',
                'watch',
            ]
            all_files = wanted_files + [
                'other-file',
            ]
            for file_name in all_files:
                file_path = os.path.join(debian_dir, file_name)
                with open(file_path, 'w') as f:
                    f.write('Contents')

            mock_cache.return_value = os.path.join(pkg_directory)

            self.run_task(initial_task=True)

            # Check that all the wanted files are created!
            self.assertEqual(len(wanted_files),
                             ExtractedSourceFile.objects.count())
            extracted_names = [
                extracted_file.name
                for extracted_file in ExtractedSourceFile.objects.all()
            ]
            for wanted_file in wanted_files:
                self.assertIn(wanted_file, extracted_names)

    @mock.patch('distro_tracker.extract_source_files.tracker_tasks.'
                'AptCache.retrieve_source')
    def test_task_is_initial_existing_files(self, mock_cache):
        """
        Tests the task when it is run as the initial task, but some files for
        the package have already been previously extracted.
        """
        name = SourcePackageName.objects.create(name='dummy-package')
        package = SourcePackage.objects.create(
            source_package_name=name, version='1.0.0')

        with make_temp_directory('dtracker-pkg-dir') as pkg_directory:
            debian_dir = os.path.join(pkg_directory, 'debian')
            os.makedirs(debian_dir)
            wanted_files = [
                'changelog',
                'copyright',
                'rules',
                'control',
                'watch',
            ]
            all_files = wanted_files + [
                'other-file',
            ]
            for file_name in all_files:
                file_path = os.path.join(debian_dir, file_name)
                with open(file_path, 'w') as f:
                    f.write('Contents')

            mock_cache.return_value = os.path.join(pkg_directory)

            # Make a previously extracted file.
            original_content = b'Original content'
            ExtractedSourceFile.objects.create(
                source_package=package,
                name='changelog',
                extracted_file=ContentFile(original_content, name='changelog'))

            self.run_task(initial_task=True)

            # Check that all the wanted files exist.
            self.assertEqual(len(wanted_files),
                             ExtractedSourceFile.objects.count())
            # Check that the existing file was not changed.
            extracted_file = ExtractedSourceFile.objects.get(
                name='changelog',
                source_package=package)
            extracted_file.extracted_file.open()
            content = extracted_file.extracted_file.read()
            extracted_file.extracted_file.close()
            self.assertEqual(original_content, content)

    @mock.patch('distro_tracker.extract_source_files.tracker_tasks.'
                'AptCache.retrieve_source')
    def test_task_is_initial_existing_file_remove(self, mock_cache):
        """
        Tests the task when it is run as the initial task, but some of the
        already extracted source files should no longer be extracted.
        """
        name = SourcePackageName.objects.create(name='dummy-package')
        package = SourcePackage.objects.create(
            source_package_name=name, version='1.0.0')

        with make_temp_directory('dtracker-pkg-dir') as pkg_directory:
            debian_dir = os.path.join(pkg_directory, 'debian')
            os.makedirs(debian_dir)
            wanted_files = [
                'changelog',
                'copyright',
                'rules',
                'control',
                'watch',
            ]
            all_files = wanted_files + [
                'other-file',
            ]
            for file_name in all_files:
                file_path = os.path.join(debian_dir, file_name)
                with open(file_path, 'w') as f:
                    f.write('Contents')

            mock_cache.return_value = os.path.join(pkg_directory)

            # Make a previously extracted file.
            original_content = 'Original content'
            ExtractedSourceFile.objects.create(
                source_package=package,
                name='we-dont-want-this-any-more',
                extracted_file=ContentFile(original_content, name='changelog'))

            self.run_task(initial_task=True)

            # Check that all the wanted files exist.
            self.assertEqual(len(wanted_files),
                             ExtractedSourceFile.objects.count())
            # Check that only the wanted files exist.
            extracted_names = [
                extracted_file.name
                for extracted_file in ExtractedSourceFile.objects.all()
            ]
            for wanted_file in wanted_files:
                self.assertIn(wanted_file, extracted_names)