Пример #1
0
    def test_perform_sync(self, curl_multi, curl):
        """
        Assert that perform_sync() makes appropriate changes to the DB and filesystem.
        """
        repo = MagicMock(spec=Repository)
        working_dir = os.path.join(self.temp_dir, "working")
        os.mkdir(working_dir)
        repo.working_dir = working_dir

        self.iso_sync_run.perform_sync()

        # There should now be three Units in the DB, but only test3.iso is the new one
        units = [tuple(call)[1][0] for call in self.sync_conduit.save_unit.mock_calls]
        self.assertEqual(len(units), 1)
        expected_unit = {'checksum': '94f7fe923212286855dea858edac1b4a292301045af0ddb275544e5251a50b3c',
                         'size': 34, 'contents': 'Are you starting to get the idea?\n', 'name': 'test3.iso'}
        unit = units[0]
        self.assertEqual(unit.unit_key['checksum'], expected_unit['checksum'])
        self.assertEqual(unit.unit_key['size'], expected_unit['size'])
        expected_storage_path = os.path.join(
            self.pkg_dir, unit.unit_key['name'], unit.unit_key['checksum'],
            str(unit.unit_key['size']), unit.unit_key['name'])
        self.assertEqual(unit.storage_path, expected_storage_path)
        with open(unit.storage_path) as data:
            contents = data.read()
        self.assertEqual(contents, expected_unit['contents'])
        # There should be 0 calls to sync_conduit.remove_unit, since remove_missing_units is False by default
        self.assertEqual(self.sync_conduit.remove_unit.call_count, 0)
Пример #2
0
    def test_perform_sync_remove_missing_units_set_false(self, curl_multi, curl):
        # Make sure the missing ISOs don't get removed if they aren't supposed to
        config = importer_mocks.get_basic_config(
            feed_url='http://fake.com/iso_feed/', max_speed=500.0, num_threads=5,
            proxy_url='http://proxy.com', proxy_port=1234, proxy_user="******",
            proxy_password='******', remove_missing_units=False,
            ssl_client_cert="Trust me, I'm who I say I am.", ssl_client_key="Secret Key",
            ssl_ca_cert="Uh, I guess that's the right server.")

        iso_sync_run = ISOSyncRun(self.sync_conduit, config)

        repo = MagicMock(spec=Repository)
        working_dir = os.path.join(self.temp_dir, "working")
        os.mkdir(working_dir)
        repo.working_dir = working_dir

        report = iso_sync_run.perform_sync()

        # There should now be three Units in the DB
        units = [tuple(call)[1][0] for call in self.sync_conduit.save_unit.mock_calls]
        self.assertEqual(len(units), 1)
        expected_unit = {'checksum': '94f7fe923212286855dea858edac1b4a292301045af0ddb275544e5251a50b3c',
                         'size': 34, 'contents': 'Are you starting to get the idea?\n', 'name': 'test3.iso'}
        unit = units[0]
        self.assertEqual(unit.unit_key['checksum'], expected_unit['checksum'])
        self.assertEqual(unit.unit_key['size'], expected_unit['size'])
        expected_storage_path = os.path.join(
            self.pkg_dir, unit.unit_key['name'], unit.unit_key['checksum'],
            str(unit.unit_key['size']), unit.unit_key['name'])
        self.assertEqual(unit.storage_path, expected_storage_path)
        with open(unit.storage_path) as data:
            contents = data.read()
        self.assertEqual(contents, expected_unit['contents'])
        # There should be 0 calls to sync_conduit.remove_unit, since remove_missing_units is False by default
        self.assertEqual(self.sync_conduit.remove_unit.call_count, 0)
    def test__symlink_units_os_error(self, readlink):
        """
        Make sure that the _symlink_units handles an OSError correctly, for the case where it doesn't raise
        EINVAL. We already have a test that raises EINVAL (test__symlink_units places an ordinary file there.)
        """
        os_error = OSError()
        # This would be an unexpected error for reading a symlink!
        os_error.errno = errno.ENOSPC
        readlink.side_effect = os_error

        repo = MagicMock(spec=Repository)
        repo.working_dir = self.temp_dir
        progress_report = progress.PublishProgressReport(self.publish_conduit)

        # There's some logic in _symlink_units to handle preexisting files and symlinks, so let's
        # create some fakes to see if it does the right thing
        build_dir = publish._get_or_create_build_dir(repo)
        unit = self.existing_units[0]
        expected_symlink_destination = os.path.join("/", "path", unit.unit_key["name"])
        os.symlink(expected_symlink_destination, os.path.join(build_dir, unit.unit_key["name"]))

        try:
            publish._symlink_units(repo, [unit], progress_report)
            self.fail("An OSError should have been raised, but was not!")
        except OSError, e:
            self.assertEqual(e.errno, errno.ENOSPC)
    def test__copy_to_hosted_location(self):
        """
        Test the operation of _copy_to_hosted_location().
        """
        repo = MagicMock(spec=Repository)
        repo.id = "lebowski"
        repo.working_dir = self.temp_dir
        progress_report = progress.PublishProgressReport(self.publish_conduit)
        config = distributor_mocks.get_basic_config(
            **{constants.CONFIG_SERVE_HTTP: True, constants.CONFIG_SERVE_HTTPS: True}
        )

        # Let's put a dummy file and a dummy symlink in the build_dir, so we can make sure they get
        # copied to the right places.
        build_dir = publish._get_or_create_build_dir(repo)
        with open(os.path.join(build_dir, "the_dude.txt"), "w") as the_dude:
            the_dude.write("Let's go bowling.")
        os.symlink("/symlink/path", os.path.join(build_dir, "symlink"))

        # This should copy our dummy file to the monkey patched folders from our setUp() method.
        publish._copy_to_hosted_location(repo, config, progress_report)

        # Make sure that the_dude.txt got copied to the right places
        expected_http_path = os.path.join(constants.ISO_HTTP_DIR, "lebowski", "the_dude.txt")
        expected_https_path = os.path.join(constants.ISO_HTTPS_DIR, "lebowski", "the_dude.txt")
        self.assertTrue(os.path.exists(expected_http_path))
        self.assertTrue(os.path.exists(expected_https_path))
        # Now make sure our symlink is also in place, and points to the correct location
        expected_http_symlink_path = os.path.join(constants.ISO_HTTP_DIR, "lebowski", "symlink")
        expected_https_symlink_path = os.path.join(constants.ISO_HTTPS_DIR, "lebowski", "symlink")
        self.assertTrue(os.path.islink(expected_http_symlink_path))
        self.assertTrue(os.path.islink(expected_https_symlink_path))
        self.assertEqual(os.path.realpath(expected_http_symlink_path), "/symlink/path")
        self.assertEqual(os.path.realpath(expected_https_symlink_path), "/symlink/path")
    def test__symlink_units_existing_correct_link(self, symlink):
        """
        Make sure that the _symlink_units handles an existing correct link well.
        """
        repo = MagicMock(spec=Repository)
        repo.working_dir = self.temp_dir
        progress_report = progress.PublishProgressReport(self.publish_conduit)

        # There's some logic in _symlink_units to handle preexisting files and symlinks, so let's
        # create some fakes to see if it does the right thing
        build_dir = publish._get_or_create_build_dir(repo)
        unit = self.existing_units[0]
        expected_symlink_destination = os.path.join("/", "path", unit.unit_key["name"])
        os.symlink(expected_symlink_destination, os.path.join(build_dir, unit.unit_key["name"]))
        # Now let's reset the Mock so that we can make sure it doesn't get called during _symlink
        symlink.reset_mock()

        publish._symlink_units(repo, [unit], progress_report)

        # The call count for symlink should be 0, because the _symlink_units call should have noticed that the
        # symlink was already correct and thus should have skipped it
        self.assertEqual(symlink.call_count, 0)
        expected_symlink_path = os.path.join(build_dir, unit.unit_key["name"])
        self.assertTrue(os.path.islink(expected_symlink_path))
        self.assertEqual(os.path.realpath(expected_symlink_path), expected_symlink_destination)
    def test_publish_repo(self):
        repo = MagicMock(spec=Repository)
        repo.id = 'lebowski'
        repo.working_dir = self.temp_dir
        publish_conduit = distributor_mocks.get_publish_conduit(existing_units=self.existing_units)
        config = distributor_mocks.get_basic_config(**{constants.CONFIG_SERVE_HTTP: True,
                                                       constants.CONFIG_SERVE_HTTPS: True})

        # We haven't implemented reporting yet, so we don't yet assert anything about the report
        # here.
        report = self.iso_distributor.publish_repo(repo, publish_conduit, config)

        # Let's verify that the publish directory looks right
        publishing_paths = [os.path.join(directory, 'lebowski') \
                          for directory in [constants.ISO_HTTP_DIR, constants.ISO_HTTPS_DIR]]
        for publishing_path in publishing_paths:
            for unit in self.existing_units:
                expected_symlink_path = os.path.join(publishing_path, unit.unit_key['name'])
                self.assertTrue(os.path.islink(expected_symlink_path))
                expected_symlink_destination = os.path.join('/', 'path', unit.unit_key['name'])
                self.assertEqual(os.path.realpath(expected_symlink_path),
                                 expected_symlink_destination)

            # Now let's have a look at the PULP_MANIFEST file to make sure it was generated and
            # published correctly.
            manifest_filename = os.path.join(publishing_path, constants.ISO_MANIFEST_FILENAME)
            manifest_rows = []
            with open(manifest_filename) as manifest_file:
                manifest = csv.reader(manifest_file)
                for row in manifest:
                    manifest_rows.append(row)
            expected_manifest_rows = [['test.iso', 'sum1', '1'], ['test2.iso', 'sum2', '2'],
                                      ['test3.iso', 'sum3', '3']]
            self.assertEqual(manifest_rows, expected_manifest_rows)
Пример #7
0
 def test_populate_branch(self):
     repo = MagicMock()
     repo.working_dir = 'aos-cd-jobs'
     repo.heads.master.commit.hexsha = '01234567abcdef'
     populate_branch(repo, 'job0')
     repo.git.add.assert_called_once_with(all=True)
     repo.index.commit.assert_called_once_with(
         'Auto-generated job branch from job0 from 0123456')
Пример #8
0
    def test_perform_sync_remove_missing_units_set_true(self, mock_download):
        mock_download.side_effect = self.fake_download

        # Make sure the missing ISOs get removed when they are supposed to
        config = importer_mocks.get_basic_config(**{
            importer_constants.KEY_FEED: 'http://fake.com/iso_feed/',
            importer_constants.KEY_MAX_SPEED: 500.0,
            importer_constants.KEY_MAX_DOWNLOADS: 5,
            importer_constants.KEY_PROXY_HOST: 'proxy.com',
            importer_constants.KEY_PROXY_PORT: 1234,
            importer_constants.KEY_PROXY_USER: "******",
            importer_constants.KEY_PROXY_PASS: '******',
            importer_constants.KEY_UNITS_REMOVE_MISSING: True,
            importer_constants.KEY_SSL_CLIENT_CERT: "Trust me, I'm who I say I am.",
            importer_constants.KEY_SSL_CLIENT_KEY: "Secret Key",
            importer_constants.KEY_SSL_CA_CERT: "Uh, I guess that's the right server.",
            importer_constants.KEY_VALIDATE: False,
        })

        self.iso_sync_run = ISOSyncRun(self.sync_conduit, config)

        repo = MagicMock(spec=Repository)
        working_dir = os.path.join(self.temp_dir, "working")
        os.mkdir(working_dir)
        repo.working_dir = working_dir

        self.iso_sync_run.perform_sync()

        # There should now be three Units in the DB
        units = [tuple(call)[1][0] for call in self.sync_conduit.save_unit.mock_calls]
        self.assertEqual(len(units), 1)
        expected_unit = {
            'checksum': '94f7fe923212286855dea858edac1b4a292301045af0ddb275544e5251a50b3c',
            'size': 34, 'contents': 'Are you starting to get the idea?\n', 'name': 'test3.iso'}
        unit = units[0]
        self.assertEqual(unit.unit_key['checksum'], expected_unit['checksum'])
        self.assertEqual(unit.unit_key['size'], expected_unit['size'])
        expected_storage_path = os.path.join(
            self.pkg_dir, unit.unit_key['name'], unit.unit_key['checksum'],
            str(unit.unit_key['size']), unit.unit_key['name'])
        self.assertEqual(unit.storage_path, expected_storage_path)

        # There should be 0 calls to sync_conduit.remove_unit, since remove_missing_units is
        # False by default
        self.assertEqual(self.sync_conduit.remove_unit.call_count, 1)
        removed_unit = self.sync_conduit.remove_unit.mock_calls[0][1][0]
        self.assertEqual(removed_unit.unit_key,
                         {'name': 'test4.iso', 'size': 4, 'checksum': 'sum4'})
    def test__get_or_create_build_dir_already_exists(self):
        """
        _get_or_create_build_dir() should correctly handle the situation when the build_dir already exists.
        """
        repo = MagicMock(spec=Repository)
        repo.working_dir = self.temp_dir

        # Assert that the build dir does not exist
        expected_build_dir = os.path.join(self.temp_dir, publish.BUILD_DIRNAME)
        os.makedirs(expected_build_dir)
        self.assertTrue(os.path.exists(expected_build_dir))

        build_dir = publish._get_or_create_build_dir(repo)

        # Assert that the build dir is correct and has been created
        self.assertEqual(build_dir, expected_build_dir)
        self.assertTrue(os.path.exists(build_dir))
Пример #10
0
    def test_perform_sync(self, progress_report, curl_multi, curl):
        """
        Assert that we perform all of the correct calls to various things during perform_sync().
        """
        repo = MagicMock(spec=Repository)
        working_dir = os.path.join(self.temp_dir, "working")
        os.mkdir(working_dir)
        pkg_dir = os.path.join(self.temp_dir, 'content')
        os.mkdir(pkg_dir)
        repo.working_dir = working_dir
        sync_conduit = importer_mocks.get_sync_conduit(type_id=TYPE_ID_ISO, pkg_dir=pkg_dir)
        config = importer_mocks.get_basic_config(
            feed_url='http://fake.com/iso_feed/', max_speed='500.0', num_threads='5',
            ssl_client_cert="Trust me, I'm who I say I am.", ssl_client_key="Secret Key",
            ssl_ca_cert="Uh, I guess that's the right server.",
            proxy_url='http://proxy.com', proxy_port='1234', proxy_user="******",
            proxy_password='******')

        report = self.iso_sync_run.perform_sync(repo, sync_conduit, config)

        # There should now be three Units in the DB, one for each of the three ISOs that our mocks
        # got us.
        units = [tuple(call)[1][0] for call in sync_conduit.save_unit.mock_calls]
        self.assertEqual(len(units), 3)
        expected_units = {
            'test.iso': {
                'checksum': 'f02d5a72cd2d57fa802840a76b44c6c6920a8b8e6b90b20e26c03876275069e0',
                'size': 16, 'contents': 'This is a file.\n'},
            'test2.iso': {
                'checksum': 'c7fbc0e821c0871805a99584c6a384533909f68a6bbe9a2a687d28d9f3b10c16',
                'size': 22, 'contents': 'This is another file.\n'},
            'test3.iso': {
                'checksum': '94f7fe923212286855dea858edac1b4a292301045af0ddb275544e5251a50b3c',
                'size': 34, 'contents': 'Are you starting to get the idea?\n'}}
        for unit in units:
            expected_unit = expected_units[unit.unit_key['name']]
            self.assertEqual(unit.unit_key['checksum'], expected_unit['checksum'])
            self.assertEqual(unit.unit_key['size'], expected_unit['size'])
            expected_storage_path = os.path.join(
                pkg_dir, unit.unit_key['name'], unit.unit_key['checksum'],
                str(unit.unit_key['size']), unit.unit_key['name'])
            self.assertEqual(unit.storage_path, expected_storage_path)
            with open(unit.storage_path) as data:
                contents = data.read()
            self.assertEqual(contents, expected_unit['contents'])
    def test__build_metadata(self):
        """
        The _build_metadata() method should put the metadata in the build directory.
        """
        repo = MagicMock(spec=Repository)
        repo.working_dir = self.temp_dir
        progress_report = progress.PublishProgressReport(self.publish_conduit)

        publish._build_metadata(repo, self.existing_units, progress_report)

        # Now let's have a look at the PULP_MANIFEST file to make sure it was generated correctly.
        manifest_filename = os.path.join(self.temp_dir, publish.BUILD_DIRNAME, constants.ISO_MANIFEST_FILENAME)
        manifest_rows = []
        with open(manifest_filename) as manifest_file:
            manifest = csv.reader(manifest_file)
            for row in manifest:
                manifest_rows.append(row)
        expected_manifest_rows = [["test.iso", "sum1", "1"], ["test2.iso", "sum2", "2"], ["test3.iso", "sum3", "3"]]
        self.assertEqual(manifest_rows, expected_manifest_rows)
    def test__get_or_create_build_dir_oserror(self, makedirs):
        """
        Let's raise the OSError that this method tries to catch to make sure we raise it appropriately.
        """
        os_error = OSError()
        os_error.errno = errno.ENOSPC
        makedirs.side_effect = os_error

        repo = MagicMock(spec=Repository)
        repo.working_dir = self.temp_dir

        # Assert that the build dir does not exist
        expected_build_dir = os.path.join(self.temp_dir, publish.BUILD_DIRNAME)
        self.assertFalse(os.path.exists(expected_build_dir))

        try:
            publish._get_or_create_build_dir(repo)
            self.fail("An OSError should have been raised but was not.")
        except OSError, e:
            self.assertEqual(e.errno, errno.ENOSPC)
    def test__get_or_create_build_dir(self):
        """
        _get_or_create_build_dir() should create the directory the first time it is called, and
        should return the path to it both times it is called.
        """
        repo = MagicMock(spec=Repository)
        repo.working_dir = self.temp_dir

        # Assert that the build dir does not exist
        expected_build_dir = os.path.join(self.temp_dir, publish.BUILD_DIRNAME)
        self.assertFalse(os.path.exists(expected_build_dir))

        build_dir = publish._get_or_create_build_dir(repo)

        # Assert that the build dir is correct and has been created
        self.assertEqual(build_dir, expected_build_dir)
        self.assertTrue(os.path.exists(build_dir))

        # It should not blow up if we call it again
        build_dir = publish._get_or_create_build_dir(repo)
        self.assertEqual(build_dir, expected_build_dir)
Пример #14
0
    def test_sorted_order(self, a):
        with self.temp_dir() as target:
            with self.temp_dir() as temp_repo:
                # Create fake env.spec and manifest files
                with open(os.path.join(temp_repo, 'env.spec'), 'w') as \
                        env_spec_file:
                    env_spec_file.write(ENV_SPEC)
                with open(os.path.join(temp_repo, 'env.manifest'), 'w') as \
                        env_manifest_file:
                    env_manifest_file.write(ENV_MANIFEST)

                # Set up arguments for call to create_rpmbuild_for_tag
                repo = MagicMock()
                repo.working_dir = temp_repo
                tag_name = 'env-default-2018_03_26'
                # Create directory that the spec file will be written to
                os.mkdir(os.path.join(target, 'SPECS'))
                create_rpmbuild_for_tag(repo, tag_name, target, CONFIG)

            # Compare the written spec file with what we'd expect
            with open(
                    os.path.join(target, 'SPECS',
                                 'SciTools-env-default-tag-2018_03_26.spec'),
                    'r') as fh:
                result_order = []
                for line in fh.readlines():
                    # Parse the lines of the spec file that are formatted as
                    # `  {INSTALL} xz-5.2.3-0\n`
                    line = line.strip()
                    if line.startswith('${INSTALL}'):
                        result_order.append(line.split(' ')[1])

            expected_order = [
                'ca-certificates-2018.1.18-0', 'ncurses-5.9-10', 'tk-8.6.7-0',
                'xz-5.2.3-0', 'zlib-1.2.11-0', 'openssl-1.0.2n-0',
                'sqlite-3.20.1-2', 'python-3.6.4-0'
            ]
            self.assertEqual(expected_order, result_order)
    def test__symlink_units(self):
        """
        Make sure that the _symlink_units creates all the correct symlinks.
        """
        repo = MagicMock(spec=Repository)
        repo.working_dir = self.temp_dir
        progress_report = progress.PublishProgressReport(self.publish_conduit)

        # There's some logic in _symlink_units to handle preexisting files and symlinks, so let's
        # create some fakes to see if it does the right thing
        build_dir = publish._get_or_create_build_dir(repo)
        os.symlink("/some/weird/path", os.path.join(build_dir, self.existing_units[0].unit_key["name"]))
        with open(os.path.join(build_dir, self.existing_units[1].unit_key["name"]), "w") as wrong:
            wrong.write("This is wrong.")

        publish._symlink_units(repo, self.existing_units, progress_report)

        build_dir = publish._get_or_create_build_dir(repo)
        for unit in self.existing_units:
            expected_symlink_path = os.path.join(build_dir, unit.unit_key["name"])
            self.assertTrue(os.path.islink(expected_symlink_path))
            expected_symlink_destination = os.path.join("/", "path", unit.unit_key["name"])
            self.assertEqual(os.path.realpath(expected_symlink_path), expected_symlink_destination)
def mock_changed_git_files(add_files):
    repo = MagicMock()
    repo.index.diff.return_value = mock_git_diff(
        [mock_a_file(f) for f in add_files])
    repo.working_dir = "."
    return repo