def test_perform_sync(self, curl_multi, curl): """ Assert that perform_sync() makes appropriate changes to the DB and filesystem. """ repo = MagicMock(spec=Repository) working_dir = os.path.join(self.temp_dir, "working") os.mkdir(working_dir) repo.working_dir = working_dir self.iso_sync_run.perform_sync() # There should now be three Units in the DB, but only test3.iso is the new one units = [tuple(call)[1][0] for call in self.sync_conduit.save_unit.mock_calls] self.assertEqual(len(units), 1) expected_unit = {'checksum': '94f7fe923212286855dea858edac1b4a292301045af0ddb275544e5251a50b3c', 'size': 34, 'contents': 'Are you starting to get the idea?\n', 'name': 'test3.iso'} unit = units[0] self.assertEqual(unit.unit_key['checksum'], expected_unit['checksum']) self.assertEqual(unit.unit_key['size'], expected_unit['size']) expected_storage_path = os.path.join( self.pkg_dir, unit.unit_key['name'], unit.unit_key['checksum'], str(unit.unit_key['size']), unit.unit_key['name']) self.assertEqual(unit.storage_path, expected_storage_path) with open(unit.storage_path) as data: contents = data.read() self.assertEqual(contents, expected_unit['contents']) # There should be 0 calls to sync_conduit.remove_unit, since remove_missing_units is False by default self.assertEqual(self.sync_conduit.remove_unit.call_count, 0)
def test_perform_sync_remove_missing_units_set_false(self, curl_multi, curl): # Make sure the missing ISOs don't get removed if they aren't supposed to config = importer_mocks.get_basic_config( feed_url='http://fake.com/iso_feed/', max_speed=500.0, num_threads=5, proxy_url='http://proxy.com', proxy_port=1234, proxy_user="******", proxy_password='******', remove_missing_units=False, ssl_client_cert="Trust me, I'm who I say I am.", ssl_client_key="Secret Key", ssl_ca_cert="Uh, I guess that's the right server.") iso_sync_run = ISOSyncRun(self.sync_conduit, config) repo = MagicMock(spec=Repository) working_dir = os.path.join(self.temp_dir, "working") os.mkdir(working_dir) repo.working_dir = working_dir report = iso_sync_run.perform_sync() # There should now be three Units in the DB units = [tuple(call)[1][0] for call in self.sync_conduit.save_unit.mock_calls] self.assertEqual(len(units), 1) expected_unit = {'checksum': '94f7fe923212286855dea858edac1b4a292301045af0ddb275544e5251a50b3c', 'size': 34, 'contents': 'Are you starting to get the idea?\n', 'name': 'test3.iso'} unit = units[0] self.assertEqual(unit.unit_key['checksum'], expected_unit['checksum']) self.assertEqual(unit.unit_key['size'], expected_unit['size']) expected_storage_path = os.path.join( self.pkg_dir, unit.unit_key['name'], unit.unit_key['checksum'], str(unit.unit_key['size']), unit.unit_key['name']) self.assertEqual(unit.storage_path, expected_storage_path) with open(unit.storage_path) as data: contents = data.read() self.assertEqual(contents, expected_unit['contents']) # There should be 0 calls to sync_conduit.remove_unit, since remove_missing_units is False by default self.assertEqual(self.sync_conduit.remove_unit.call_count, 0)
def test__symlink_units_os_error(self, readlink): """ Make sure that the _symlink_units handles an OSError correctly, for the case where it doesn't raise EINVAL. We already have a test that raises EINVAL (test__symlink_units places an ordinary file there.) """ os_error = OSError() # This would be an unexpected error for reading a symlink! os_error.errno = errno.ENOSPC readlink.side_effect = os_error repo = MagicMock(spec=Repository) repo.working_dir = self.temp_dir progress_report = progress.PublishProgressReport(self.publish_conduit) # There's some logic in _symlink_units to handle preexisting files and symlinks, so let's # create some fakes to see if it does the right thing build_dir = publish._get_or_create_build_dir(repo) unit = self.existing_units[0] expected_symlink_destination = os.path.join("/", "path", unit.unit_key["name"]) os.symlink(expected_symlink_destination, os.path.join(build_dir, unit.unit_key["name"])) try: publish._symlink_units(repo, [unit], progress_report) self.fail("An OSError should have been raised, but was not!") except OSError, e: self.assertEqual(e.errno, errno.ENOSPC)
def test__copy_to_hosted_location(self): """ Test the operation of _copy_to_hosted_location(). """ repo = MagicMock(spec=Repository) repo.id = "lebowski" repo.working_dir = self.temp_dir progress_report = progress.PublishProgressReport(self.publish_conduit) config = distributor_mocks.get_basic_config( **{constants.CONFIG_SERVE_HTTP: True, constants.CONFIG_SERVE_HTTPS: True} ) # Let's put a dummy file and a dummy symlink in the build_dir, so we can make sure they get # copied to the right places. build_dir = publish._get_or_create_build_dir(repo) with open(os.path.join(build_dir, "the_dude.txt"), "w") as the_dude: the_dude.write("Let's go bowling.") os.symlink("/symlink/path", os.path.join(build_dir, "symlink")) # This should copy our dummy file to the monkey patched folders from our setUp() method. publish._copy_to_hosted_location(repo, config, progress_report) # Make sure that the_dude.txt got copied to the right places expected_http_path = os.path.join(constants.ISO_HTTP_DIR, "lebowski", "the_dude.txt") expected_https_path = os.path.join(constants.ISO_HTTPS_DIR, "lebowski", "the_dude.txt") self.assertTrue(os.path.exists(expected_http_path)) self.assertTrue(os.path.exists(expected_https_path)) # Now make sure our symlink is also in place, and points to the correct location expected_http_symlink_path = os.path.join(constants.ISO_HTTP_DIR, "lebowski", "symlink") expected_https_symlink_path = os.path.join(constants.ISO_HTTPS_DIR, "lebowski", "symlink") self.assertTrue(os.path.islink(expected_http_symlink_path)) self.assertTrue(os.path.islink(expected_https_symlink_path)) self.assertEqual(os.path.realpath(expected_http_symlink_path), "/symlink/path") self.assertEqual(os.path.realpath(expected_https_symlink_path), "/symlink/path")
def test__symlink_units_existing_correct_link(self, symlink): """ Make sure that the _symlink_units handles an existing correct link well. """ repo = MagicMock(spec=Repository) repo.working_dir = self.temp_dir progress_report = progress.PublishProgressReport(self.publish_conduit) # There's some logic in _symlink_units to handle preexisting files and symlinks, so let's # create some fakes to see if it does the right thing build_dir = publish._get_or_create_build_dir(repo) unit = self.existing_units[0] expected_symlink_destination = os.path.join("/", "path", unit.unit_key["name"]) os.symlink(expected_symlink_destination, os.path.join(build_dir, unit.unit_key["name"])) # Now let's reset the Mock so that we can make sure it doesn't get called during _symlink symlink.reset_mock() publish._symlink_units(repo, [unit], progress_report) # The call count for symlink should be 0, because the _symlink_units call should have noticed that the # symlink was already correct and thus should have skipped it self.assertEqual(symlink.call_count, 0) expected_symlink_path = os.path.join(build_dir, unit.unit_key["name"]) self.assertTrue(os.path.islink(expected_symlink_path)) self.assertEqual(os.path.realpath(expected_symlink_path), expected_symlink_destination)
def test_publish_repo(self): repo = MagicMock(spec=Repository) repo.id = 'lebowski' repo.working_dir = self.temp_dir publish_conduit = distributor_mocks.get_publish_conduit(existing_units=self.existing_units) config = distributor_mocks.get_basic_config(**{constants.CONFIG_SERVE_HTTP: True, constants.CONFIG_SERVE_HTTPS: True}) # We haven't implemented reporting yet, so we don't yet assert anything about the report # here. report = self.iso_distributor.publish_repo(repo, publish_conduit, config) # Let's verify that the publish directory looks right publishing_paths = [os.path.join(directory, 'lebowski') \ for directory in [constants.ISO_HTTP_DIR, constants.ISO_HTTPS_DIR]] for publishing_path in publishing_paths: for unit in self.existing_units: expected_symlink_path = os.path.join(publishing_path, unit.unit_key['name']) self.assertTrue(os.path.islink(expected_symlink_path)) expected_symlink_destination = os.path.join('/', 'path', unit.unit_key['name']) self.assertEqual(os.path.realpath(expected_symlink_path), expected_symlink_destination) # Now let's have a look at the PULP_MANIFEST file to make sure it was generated and # published correctly. manifest_filename = os.path.join(publishing_path, constants.ISO_MANIFEST_FILENAME) manifest_rows = [] with open(manifest_filename) as manifest_file: manifest = csv.reader(manifest_file) for row in manifest: manifest_rows.append(row) expected_manifest_rows = [['test.iso', 'sum1', '1'], ['test2.iso', 'sum2', '2'], ['test3.iso', 'sum3', '3']] self.assertEqual(manifest_rows, expected_manifest_rows)
def test_populate_branch(self): repo = MagicMock() repo.working_dir = 'aos-cd-jobs' repo.heads.master.commit.hexsha = '01234567abcdef' populate_branch(repo, 'job0') repo.git.add.assert_called_once_with(all=True) repo.index.commit.assert_called_once_with( 'Auto-generated job branch from job0 from 0123456')
def test_perform_sync_remove_missing_units_set_true(self, mock_download): mock_download.side_effect = self.fake_download # Make sure the missing ISOs get removed when they are supposed to config = importer_mocks.get_basic_config(**{ importer_constants.KEY_FEED: 'http://fake.com/iso_feed/', importer_constants.KEY_MAX_SPEED: 500.0, importer_constants.KEY_MAX_DOWNLOADS: 5, importer_constants.KEY_PROXY_HOST: 'proxy.com', importer_constants.KEY_PROXY_PORT: 1234, importer_constants.KEY_PROXY_USER: "******", importer_constants.KEY_PROXY_PASS: '******', importer_constants.KEY_UNITS_REMOVE_MISSING: True, importer_constants.KEY_SSL_CLIENT_CERT: "Trust me, I'm who I say I am.", importer_constants.KEY_SSL_CLIENT_KEY: "Secret Key", importer_constants.KEY_SSL_CA_CERT: "Uh, I guess that's the right server.", importer_constants.KEY_VALIDATE: False, }) self.iso_sync_run = ISOSyncRun(self.sync_conduit, config) repo = MagicMock(spec=Repository) working_dir = os.path.join(self.temp_dir, "working") os.mkdir(working_dir) repo.working_dir = working_dir self.iso_sync_run.perform_sync() # There should now be three Units in the DB units = [tuple(call)[1][0] for call in self.sync_conduit.save_unit.mock_calls] self.assertEqual(len(units), 1) expected_unit = { 'checksum': '94f7fe923212286855dea858edac1b4a292301045af0ddb275544e5251a50b3c', 'size': 34, 'contents': 'Are you starting to get the idea?\n', 'name': 'test3.iso'} unit = units[0] self.assertEqual(unit.unit_key['checksum'], expected_unit['checksum']) self.assertEqual(unit.unit_key['size'], expected_unit['size']) expected_storage_path = os.path.join( self.pkg_dir, unit.unit_key['name'], unit.unit_key['checksum'], str(unit.unit_key['size']), unit.unit_key['name']) self.assertEqual(unit.storage_path, expected_storage_path) # There should be 0 calls to sync_conduit.remove_unit, since remove_missing_units is # False by default self.assertEqual(self.sync_conduit.remove_unit.call_count, 1) removed_unit = self.sync_conduit.remove_unit.mock_calls[0][1][0] self.assertEqual(removed_unit.unit_key, {'name': 'test4.iso', 'size': 4, 'checksum': 'sum4'})
def test__get_or_create_build_dir_already_exists(self): """ _get_or_create_build_dir() should correctly handle the situation when the build_dir already exists. """ repo = MagicMock(spec=Repository) repo.working_dir = self.temp_dir # Assert that the build dir does not exist expected_build_dir = os.path.join(self.temp_dir, publish.BUILD_DIRNAME) os.makedirs(expected_build_dir) self.assertTrue(os.path.exists(expected_build_dir)) build_dir = publish._get_or_create_build_dir(repo) # Assert that the build dir is correct and has been created self.assertEqual(build_dir, expected_build_dir) self.assertTrue(os.path.exists(build_dir))
def test_perform_sync(self, progress_report, curl_multi, curl): """ Assert that we perform all of the correct calls to various things during perform_sync(). """ repo = MagicMock(spec=Repository) working_dir = os.path.join(self.temp_dir, "working") os.mkdir(working_dir) pkg_dir = os.path.join(self.temp_dir, 'content') os.mkdir(pkg_dir) repo.working_dir = working_dir sync_conduit = importer_mocks.get_sync_conduit(type_id=TYPE_ID_ISO, pkg_dir=pkg_dir) config = importer_mocks.get_basic_config( feed_url='http://fake.com/iso_feed/', max_speed='500.0', num_threads='5', ssl_client_cert="Trust me, I'm who I say I am.", ssl_client_key="Secret Key", ssl_ca_cert="Uh, I guess that's the right server.", proxy_url='http://proxy.com', proxy_port='1234', proxy_user="******", proxy_password='******') report = self.iso_sync_run.perform_sync(repo, sync_conduit, config) # There should now be three Units in the DB, one for each of the three ISOs that our mocks # got us. units = [tuple(call)[1][0] for call in sync_conduit.save_unit.mock_calls] self.assertEqual(len(units), 3) expected_units = { 'test.iso': { 'checksum': 'f02d5a72cd2d57fa802840a76b44c6c6920a8b8e6b90b20e26c03876275069e0', 'size': 16, 'contents': 'This is a file.\n'}, 'test2.iso': { 'checksum': 'c7fbc0e821c0871805a99584c6a384533909f68a6bbe9a2a687d28d9f3b10c16', 'size': 22, 'contents': 'This is another file.\n'}, 'test3.iso': { 'checksum': '94f7fe923212286855dea858edac1b4a292301045af0ddb275544e5251a50b3c', 'size': 34, 'contents': 'Are you starting to get the idea?\n'}} for unit in units: expected_unit = expected_units[unit.unit_key['name']] self.assertEqual(unit.unit_key['checksum'], expected_unit['checksum']) self.assertEqual(unit.unit_key['size'], expected_unit['size']) expected_storage_path = os.path.join( pkg_dir, unit.unit_key['name'], unit.unit_key['checksum'], str(unit.unit_key['size']), unit.unit_key['name']) self.assertEqual(unit.storage_path, expected_storage_path) with open(unit.storage_path) as data: contents = data.read() self.assertEqual(contents, expected_unit['contents'])
def test__build_metadata(self): """ The _build_metadata() method should put the metadata in the build directory. """ repo = MagicMock(spec=Repository) repo.working_dir = self.temp_dir progress_report = progress.PublishProgressReport(self.publish_conduit) publish._build_metadata(repo, self.existing_units, progress_report) # Now let's have a look at the PULP_MANIFEST file to make sure it was generated correctly. manifest_filename = os.path.join(self.temp_dir, publish.BUILD_DIRNAME, constants.ISO_MANIFEST_FILENAME) manifest_rows = [] with open(manifest_filename) as manifest_file: manifest = csv.reader(manifest_file) for row in manifest: manifest_rows.append(row) expected_manifest_rows = [["test.iso", "sum1", "1"], ["test2.iso", "sum2", "2"], ["test3.iso", "sum3", "3"]] self.assertEqual(manifest_rows, expected_manifest_rows)
def test__get_or_create_build_dir_oserror(self, makedirs): """ Let's raise the OSError that this method tries to catch to make sure we raise it appropriately. """ os_error = OSError() os_error.errno = errno.ENOSPC makedirs.side_effect = os_error repo = MagicMock(spec=Repository) repo.working_dir = self.temp_dir # Assert that the build dir does not exist expected_build_dir = os.path.join(self.temp_dir, publish.BUILD_DIRNAME) self.assertFalse(os.path.exists(expected_build_dir)) try: publish._get_or_create_build_dir(repo) self.fail("An OSError should have been raised but was not.") except OSError, e: self.assertEqual(e.errno, errno.ENOSPC)
def test__get_or_create_build_dir(self): """ _get_or_create_build_dir() should create the directory the first time it is called, and should return the path to it both times it is called. """ repo = MagicMock(spec=Repository) repo.working_dir = self.temp_dir # Assert that the build dir does not exist expected_build_dir = os.path.join(self.temp_dir, publish.BUILD_DIRNAME) self.assertFalse(os.path.exists(expected_build_dir)) build_dir = publish._get_or_create_build_dir(repo) # Assert that the build dir is correct and has been created self.assertEqual(build_dir, expected_build_dir) self.assertTrue(os.path.exists(build_dir)) # It should not blow up if we call it again build_dir = publish._get_or_create_build_dir(repo) self.assertEqual(build_dir, expected_build_dir)
def test_sorted_order(self, a): with self.temp_dir() as target: with self.temp_dir() as temp_repo: # Create fake env.spec and manifest files with open(os.path.join(temp_repo, 'env.spec'), 'w') as \ env_spec_file: env_spec_file.write(ENV_SPEC) with open(os.path.join(temp_repo, 'env.manifest'), 'w') as \ env_manifest_file: env_manifest_file.write(ENV_MANIFEST) # Set up arguments for call to create_rpmbuild_for_tag repo = MagicMock() repo.working_dir = temp_repo tag_name = 'env-default-2018_03_26' # Create directory that the spec file will be written to os.mkdir(os.path.join(target, 'SPECS')) create_rpmbuild_for_tag(repo, tag_name, target, CONFIG) # Compare the written spec file with what we'd expect with open( os.path.join(target, 'SPECS', 'SciTools-env-default-tag-2018_03_26.spec'), 'r') as fh: result_order = [] for line in fh.readlines(): # Parse the lines of the spec file that are formatted as # ` {INSTALL} xz-5.2.3-0\n` line = line.strip() if line.startswith('${INSTALL}'): result_order.append(line.split(' ')[1]) expected_order = [ 'ca-certificates-2018.1.18-0', 'ncurses-5.9-10', 'tk-8.6.7-0', 'xz-5.2.3-0', 'zlib-1.2.11-0', 'openssl-1.0.2n-0', 'sqlite-3.20.1-2', 'python-3.6.4-0' ] self.assertEqual(expected_order, result_order)
def test__symlink_units(self): """ Make sure that the _symlink_units creates all the correct symlinks. """ repo = MagicMock(spec=Repository) repo.working_dir = self.temp_dir progress_report = progress.PublishProgressReport(self.publish_conduit) # There's some logic in _symlink_units to handle preexisting files and symlinks, so let's # create some fakes to see if it does the right thing build_dir = publish._get_or_create_build_dir(repo) os.symlink("/some/weird/path", os.path.join(build_dir, self.existing_units[0].unit_key["name"])) with open(os.path.join(build_dir, self.existing_units[1].unit_key["name"]), "w") as wrong: wrong.write("This is wrong.") publish._symlink_units(repo, self.existing_units, progress_report) build_dir = publish._get_or_create_build_dir(repo) for unit in self.existing_units: expected_symlink_path = os.path.join(build_dir, unit.unit_key["name"]) self.assertTrue(os.path.islink(expected_symlink_path)) expected_symlink_destination = os.path.join("/", "path", unit.unit_key["name"]) self.assertEqual(os.path.realpath(expected_symlink_path), expected_symlink_destination)
def mock_changed_git_files(add_files): repo = MagicMock() repo.index.diff.return_value = mock_git_diff( [mock_a_file(f) for f in add_files]) repo.working_dir = "." return repo