def test_process_main_only_publish_directory_contents(self): source_dir = os.path.join(self.working_directory, 'source') master_dir = os.path.join(self.working_directory, 'master') publish_dir = os.path.join(self.working_directory, 'publish', 'bar') publish_dir += '/' step = publish_step.AtomicDirectoryPublishStep( source_dir, [('/', publish_dir)], master_dir, only_publish_directory_contents=True) step.parent = Mock(timestamp=str(time.time())) # create some files to test sub_file = os.path.join(source_dir, 'bar.html') touch(sub_file) # create an existing file that will be maintained existing_file = os.path.join(source_dir, 'bar.html') touch(existing_file) # Create an old directory to test old_dir = os.path.join(master_dir, 'foo') os.makedirs(old_dir) step.process_main() target_file = os.path.join(publish_dir, 'bar.html') self.assertEquals(True, os.path.exists(target_file)) self.assertTrue(os.path.exists(existing_file)) self.assertEquals(1, len(os.listdir(master_dir)))
def test_create_symlink_link_exists_not_link(self): source_path = os.path.join(self.working_dir, 'source') link_path = os.path.join(self.published_dir, 'link') touch(source_path) touch(link_path) self.assertRaises(RuntimeError, PublishStep._create_symlink, source_path, link_path)
def test_create_symlink(self): source_path = os.path.join(self.working_dir, "source") link_path = os.path.join(self.published_dir, "link") touch(source_path) self.assertFalse(os.path.exists(link_path)) misc.create_symlink(source_path, link_path) self.assertTrue(os.path.exists(link_path))
def test_create_symlink_link_exists_not_link(self): source_path = os.path.join(self.working_dir, "source") link_path = os.path.join(self.published_dir, "link") touch(source_path) touch(link_path) self.assertRaises(RuntimeError, misc.create_symlink, source_path, link_path)
def existing_files_saved(self): existing_file = os.path.join(self.destination_dir, 'foo.txt') touch(existing_file) new_dir = os.path.join(self.source_dir, 'bar') os.makedirs(new_dir) installdistributor.PuppetModuleInstallDistributor.\ _move_to_destination_directory(self.source_dir, self.destination_dir) self.assertTrue(os.path.exists(existing_file))
def test_clean_ophaned_leaf(self): """ Test that an orphaned leaf is removed. """ leaf = os.path.join(self.publish_base, 'a', 'b', 'c', 'listing') leaf_dir = os.path.dirname(leaf) util.touch(leaf) self.assertTrue(os.path.isfile(leaf)) migration.clean_simple_hosting_directories(leaf_dir, self.publish_base) self.assertFalse(os.path.isdir(os.path.join(self.publish_base, 'a')))
def test_walk_does_not_recognize_non_leaf(self, mock_clean): """ Test that non orphaned leafs are not cleaned. """ non_orphan = os.path.join(self.publish_base, 'not', 'orphan', 'listing') other_file = os.path.join(self.publish_base, 'not', 'orphan', 'notlisting') util.touch(non_orphan) util.touch(other_file) migration.walk_and_clean_directories(self.publish_base) self.assertEqual(mock_clean.call_count, 0)
def test_create_symlink_no_link_parent(self): source_path = os.path.join(self.working_dir, 'source') link_path = os.path.join(self.published_dir, 'foo/bar/baz/link') touch(source_path) self.assertFalse(os.path.exists(os.path.dirname(link_path))) PublishStep._create_symlink(source_path, link_path) self.assertTrue(os.path.exists(link_path))
def test_clean_with_concurrent_file_creation(self, mock_rmdir, mock_util): """ Clean directories when a dir cannot be removed during orphaned directory removal. """ mock_rmdir.side_effect = OSError() listing_file_a = os.path.join(self.publish_base, 'a', 'listing') updir = os.path.join(self.publish_base, 'a') util.touch(listing_file_a) old_symlink = os.path.join(self.publish_base, 'a', 'path_to_removed_symlink') self.distributor.clean_simple_hosting_directories(old_symlink, self.publish_base) mock_util.generate_listing_files.assert_called_once_with(updir, updir)
def test_create_symlink_no_link_parent_with_permissions(self, mock_makedirs, mock_symlink): source_path = os.path.join(self.working_dir, 'source') link_path = os.path.join(self.published_dir, 'foo/bar/baz/link') touch(source_path) self.assertFalse(os.path.exists(os.path.dirname(link_path))) misc.create_symlink(source_path, link_path, directory_permissions=0700) mock_makedirs.assert_called_once_with(os.path.dirname(link_path), mode=0700) mock_symlink.assert_called_once_with(source_path, link_path)
def test_create_symlink_no_link_parent(self, mock_makedirs, mock_symlink): source_path = os.path.join(self.working_dir, 'source') link_path = os.path.join(self.published_dir, 'foo/bar/baz/link') touch(source_path) self.assertFalse(os.path.exists(os.path.dirname(link_path))) misc.create_symlink(source_path, link_path) mock_makedirs.assert_called_once_with(os.path.dirname(link_path), mode=0770) mock_symlink.assert_called_once_with(source_path, link_path)
def test_clear_directory(self): for file_name in ('one', 'two', 'three'): touch(os.path.join(self.working_dir, file_name)) os.makedirs(os.path.join(self.working_dir, 'four')) self.assertEqual(len(os.listdir(self.working_dir)), 4) misc.clear_directory(self.working_dir, ['two']) self.assertEqual(len(os.listdir(self.working_dir)), 1)
def test_create_symlink_link_parent_bad_permissions(self): source_path = os.path.join(self.working_dir, 'source') link_path = os.path.join(self.published_dir, 'foo/bar/baz/link') touch(source_path) os.makedirs(os.path.dirname(link_path)) os.chmod(os.path.dirname(link_path), 0000) self.assertRaises(OSError, PublishStep._create_symlink, source_path, link_path) os.chmod(os.path.dirname(link_path), 0777)
def test_clear_directory(self): for file_name in ("one", "two", "three"): touch(os.path.join(self.working_dir, file_name)) os.makedirs(os.path.join(self.working_dir, "four")) self.assertEqual(len(os.listdir(self.working_dir)), 4) misc.clear_directory(self.working_dir, ["two"]) self.assertEqual(len(os.listdir(self.working_dir)), 1)
def test_process_unit(self): step = publish_steps.PublishImagesStep() fake_image_filename = 'fake-zero-byte-image.qcow2' touch(os.path.join(self.content_directory, fake_image_filename)) unit = Mock(unit_key={'image_checksum': 'd41d8cd98f00b204e9800998ecf8427e'}, storage_path=os.path.join(self.content_directory, fake_image_filename)) step.get_working_dir = Mock(return_value=self.publish_directory) step.process_unit(unit) # verify symlink expected_symlink = os.path.join(self.publish_directory, 'web', fake_image_filename) self.assertTrue(os.path.exists(expected_symlink))
def test_create_symlink_no_link_parent_with_permissions(self, mock_makedirs, mock_symlink): source_path = os.path.join(self.working_dir, "source") link_path = os.path.join(self.published_dir, "foo/bar/baz/link") touch(source_path) self.assertFalse(os.path.exists(os.path.dirname(link_path))) misc.create_symlink(source_path, link_path, directory_permissions=0700) mock_makedirs.assert_called_once_with(os.path.dirname(link_path), mode=0700) mock_symlink.assert_called_once_with(source_path, link_path)
def test_create_symlink(self): source_path = os.path.join(self.working_dir, 'source') link_path = os.path.join(self.published_dir, 'link') touch(source_path) self.assertFalse(os.path.exists(link_path)) PublishStep._create_symlink(source_path, link_path) self.assertTrue(os.path.exists(link_path)) self.assertTrue(os.path.islink(link_path)) self.assertEqual(os.readlink(link_path), source_path)
def test_distributor_removed(self, mock_repo_dir): mock_repo_dir.return_value = os.path.join(self.working_dir, 'repo') os.makedirs(mock_repo_dir.return_value) working_dir = os.path.join(self.working_dir, 'working') repo = Mock(id='bar', working_dir=working_dir) config = {} touch(os.path.join(working_dir, 'bar.json')) touch(os.path.join(mock_repo_dir.return_value, 'bar.tar')) self.distributor.distributor_removed(repo, config) self.assertEquals(0, len(os.listdir(mock_repo_dir.return_value))) self.assertEquals(1, len(os.listdir(self.working_dir)))
def test_distributor_removed(self, mock_web, mock_master, mock_app, m_repo_objects): m_repo_objects.get_repo_or_missing_resource.return_value = Mock(repo_id='bar') mock_app.return_value = os.path.join(self.working_dir) mock_web.return_value = os.path.join(self.working_dir, 'web') mock_master.return_value = os.path.join(self.working_dir, 'master') os.makedirs(mock_web.return_value) os.makedirs(mock_master.return_value) repo = Mock(id='bar') config = {} touch(os.path.join(self.working_dir, 'bar.json')) self.distributor.distributor_removed(repo, config) self.assertEquals(0, len(os.listdir(self.working_dir)))
def test_create_symlink_link_exists_and_is_correct(self): new_source_path = os.path.join(self.working_dir, "new_source") link_path = os.path.join(self.published_dir, "link") touch(new_source_path) os.symlink(new_source_path, link_path) self.assertEqual(os.readlink(link_path), new_source_path) misc.create_symlink(new_source_path, link_path) self.assertEqual(os.readlink(link_path), new_source_path)
def test_process_main(self): source_dir = os.path.join(self.working_directory, 'source') os.makedirs(source_dir) target_file = os.path.join(self.working_directory, 'target', 'target.tar') step = publish_step.SaveTarFilePublishStep(source_dir, target_file) touch(os.path.join(source_dir, 'foo.txt')) step.process_main() with contextlib.closing(tarfile.open(target_file)) as tar_file: names = tar_file.getnames() # the first item is either '' or '.' depending on if this is py2.7 or py2.6 self.assertEquals(names[1:], ['foo.txt'])
def test_create_symlink_link_exists_and_is_correct(self): new_source_path = os.path.join(self.working_dir, 'new_source') link_path = os.path.join(self.published_dir, 'link') touch(new_source_path) os.symlink(new_source_path, link_path) self.assertEqual(os.readlink(link_path), new_source_path) PublishStep._create_symlink(new_source_path, link_path) self.assertEqual(os.readlink(link_path), new_source_path)
def test_process_main(self): source_dir = os.path.join(self.working_directory, 'source') os.makedirs(source_dir) touch(os.path.join(source_dir, 'foo.txt')) target_dir = os.path.join(self.working_directory, 'target') target_file = os.path.join(target_dir, 'foo.txt') step = publish_step.CopyDirectoryStep(source_dir, target_dir) touch(os.path.join(source_dir, 'foo.txt')) step.process_main() self.assertTrue(os.path.exists(target_file))
def test_process_units(self): step = publish_steps.PublishImagesStep() step.parent = self.parent step.redirect_context = Mock() file_list = ['ancestry', 'layer', 'json'] for file_name in file_list: touch(os.path.join(self.content_directory, file_name)) unit = Mock(unit_key={'image_id': 'foo_image'}, storage_path=self.content_directory) step.get_working_dir = Mock(return_value=self.publish_directory) step.process_unit(unit) step.redirect_context.add_unit_metadata.assert_called_once_with(unit) for file_name in file_list: self.assertTrue(os.path.exists(os.path.join(self.publish_directory, 'web', 'foo_image', file_name)))
def test_distributor_removed(self, mock_web, mock_master, mock_app): mock_app.return_value = os.path.join(self.working_dir) mock_web.return_value = os.path.join(self.working_dir, 'web') mock_master.return_value = os.path.join(self.working_dir, 'master') working_dir = os.path.join(self.working_dir, 'working') os.makedirs(mock_web.return_value) os.makedirs(mock_master.return_value) repo = Mock(id='bar', working_dir=working_dir) config = {} touch(os.path.join(self.working_dir, 'bar.json')) self.distributor.distributor_removed(repo, config) self.assertEquals(0, len(os.listdir(self.working_dir)))
def test_create_symlink_link_exists(self): old_source_path = os.path.join(self.working_dir, "old_source") new_source_path = os.path.join(self.working_dir, "new_source") link_path = os.path.join(self.published_dir, "link") touch(old_source_path) touch(new_source_path) os.symlink(old_source_path, link_path) self.assertEqual(os.readlink(link_path), old_source_path) link_path_with_slash = link_path + "/" misc.create_symlink(new_source_path, link_path_with_slash) self.assertEqual(os.readlink(link_path), new_source_path)
def test_create_symlink_link_exists(self): old_source_path = os.path.join(self.working_dir, 'old_source') new_source_path = os.path.join(self.working_dir, 'new_source') link_path = os.path.join(self.published_dir, 'link') touch(old_source_path) touch(new_source_path) os.symlink(old_source_path, link_path) self.assertEqual(os.readlink(link_path), old_source_path) link_path_with_slash = link_path + '/' PublishStep._create_symlink(new_source_path, link_path_with_slash) self.assertEqual(os.readlink(link_path), new_source_path)
def test_process_units(self): step = publish_steps.PublishImagesStep() step.parent = self.parent step.redirect_context = Mock() file_list = ['ancestry', 'layer', 'json'] for file_name in file_list: touch(os.path.join(self.content_directory, file_name)) unit = Mock(unit_key={'image_id': 'foo_image'}, storage_path=self.content_directory) step.get_working_dir = Mock(return_value=self.publish_directory) step.process_unit(unit) step.redirect_context.add_unit_metadata.assert_called_once_with(unit) for file_name in file_list: self.assertTrue( os.path.exists( os.path.join(self.publish_directory, 'web', 'foo_image', file_name)))
def test_process_main(self, mock_get_unit): mock_get_unit.return_value.storage_path = self.source_dir content_dirs = ['objects', 'remote-cache', 'tmp', 'uncompressed-objects-cache'] for dir_name in content_dirs: touch(os.path.join(self.source_dir, dir_name, 'foo')) touch(os.path.join(self.source_dir, 'refs', 'foo')) step = steps.PublishContentStep() self.parent.add_child(step) step.process_main() for dir_name in content_dirs: target_dir = os.path.join(self.target_dir, dir_name) src_dir = os.path.join(self.source_dir, dir_name) self.assertEquals(os.path.realpath(target_dir), src_dir) self.assertFalse(os.path.exists(os.path.join(self.target_dir, 'refs')))
def test_clean_only_ophaned_leaf(self): """ Test partially shared path, only unshared portion of ophan path should be removed. """ listing_file_a = os.path.join(self.publish_base, 'a', 'listing') listing_file_b = os.path.join(self.publish_base, 'a', 'b', 'listing') listing_file_c = os.path.join(self.publish_base, 'a', 'b', 'c', 'listing') non_orphan_listing = os.path.join(self.publish_base, 'a', 'other', 'listing') non_orphan_file = os.path.join(self.publish_base, 'a', 'other', 'otherfile') util.touch(listing_file_a) util.touch(listing_file_b) util.touch(listing_file_c) util.touch(non_orphan_listing) util.touch(non_orphan_file) old_symlink = os.path.join(self.publish_base, 'a', 'b', 'c', 'path_to_removed_symlink') self.distributor.clean_simple_hosting_directories(old_symlink, self.publish_base) self.assertTrue(os.path.isdir(os.path.join(self.publish_base, 'a'))) self.assertFalse(os.path.isdir(os.path.join(self.publish_base, 'a', 'b')))
def test_pre_save_signal_directory_content(self, mock_config): source_dir = os.path.join(self.working_dir, 'src') target_dir = os.path.join(self.working_dir, 'target') os.mkdir(source_dir) os.mkdir(target_dir) mock_config.config.get.return_value = target_dir # create something in the source directory to test the copy touch(os.path.join(source_dir, 'foo', 'bar')) class ContentUnitHelper(model.ContentUnit): unit_type_id = 'foo_unit' foo = ContentUnitHelper() foo.set_content(source_dir) foo.id = 'abbcd' model.ContentUnit.pre_save_signal(object(), foo) full_path = os.path.join(target_dir, 'units', 'foo_unit', 'a', 'bb', 'abbcd', 'foo', 'bar') self.assertTrue(os.path.exists(full_path))
def test_process_unit(self): step = publish.PublishRpmAndDrpmStepIncremental() self.publisher.add_child(step) unit_key = {'name': 'foo', 'version': '1', 'release': '2', 'arch': 'flux'} metadata = {'filename': 'bar.txt', 'repodata': 'baz', '_test': 'hidden'} original_metadata = metadata.copy() storage_path = os.path.join(self.working_dir, 'foo') touch(storage_path) test_unit = Unit('foo_type', unit_key, metadata, storage_path) step.process_unit(test_unit) original_metadata.pop('repodata') original_metadata.pop('_test') unit_file = os.path.join(self.working_dir, 'foo-1-2.flux.json') self.assertTrue(os.path.exists(unit_file)) with open(unit_file) as file_handle: loaded = json.load(file_handle) compare_dict(loaded, { 'unit_key': unit_key, 'unit_metadata': original_metadata })
def test_clean_ophaned_leaf(self): """ Test that an orphaned leaf is removed. """ listing_file_a = os.path.join(self.publish_base, 'a', 'listing') listing_file_b = os.path.join(self.publish_base, 'a', 'b', 'listing') listing_file_c = os.path.join(self.publish_base, 'a', 'b', 'c', 'listing') util.touch(listing_file_a) util.touch(listing_file_b) util.touch(listing_file_c) old_symlink = os.path.join(self.publish_base, 'a', 'b', 'c', 'path_to_removed_symlink') self.distributor.clean_simple_hosting_directories(old_symlink, self.publish_base) self.assertFalse(os.path.isdir(os.path.join(self.publish_base, 'a')))
def test_walk_recognizes_leaf(self, mock_clean): """ Test that an orphaned leaf is detected. """ leaf = os.path.join(self.publish_base, 'a', 'b', 'c', 'listing') non_orphan = os.path.join(self.publish_base, 'not', 'leaf', 'listing') other_file = os.path.join(self.publish_base, 'not', 'leaf', 'notlisting') util.touch(leaf) util.touch(non_orphan) util.touch(other_file) migration.walk_and_clean_directories(self.publish_base) expected_leaf = os.path.join(self.publish_base, 'a', 'b', 'c') mock_clean.assert_called_once_with(expected_leaf, self.publish_base)