def test_process_main_only_publish_directory_contents(self):
source_dir = os.path.join(self.working_directory, 'source')
master_dir = os.path.join(self.working_directory, 'master')
publish_dir = os.path.join(self.working_directory, 'publish', 'bar')
publish_dir += '/'
step = publish_step.AtomicDirectoryPublishStep(
source_dir, [('/', publish_dir)], master_dir, only_publish_directory_contents=True)
step.parent = Mock(timestamp=str(time.time()))
# create some files to test
sub_file = os.path.join(source_dir, 'bar.html')
touch(sub_file)
# create an existing file that will be maintained
existing_file = os.path.join(source_dir, 'bar.html')
touch(existing_file)
# Create an old directory to test
old_dir = os.path.join(master_dir, 'foo')
os.makedirs(old_dir)
step.process_main()
target_file = os.path.join(publish_dir, 'bar.html')
self.assertEquals(True, os.path.exists(target_file))
self.assertTrue(os.path.exists(existing_file))
self.assertEquals(1, len(os.listdir(master_dir)))
def test_create_symlink_link_exists_not_link(self):
source_path = os.path.join(self.working_dir, 'source')
link_path = os.path.join(self.published_dir, 'link')
touch(source_path)
touch(link_path)
self.assertRaises(RuntimeError, PublishStep._create_symlink, source_path, link_path)
def test_create_symlink(self):
source_path = os.path.join(self.working_dir, "source")
link_path = os.path.join(self.published_dir, "link")
touch(source_path)
self.assertFalse(os.path.exists(link_path))
misc.create_symlink(source_path, link_path)
self.assertTrue(os.path.exists(link_path))
def test_create_symlink_link_exists_not_link(self):
source_path = os.path.join(self.working_dir, "source")
link_path = os.path.join(self.published_dir, "link")
touch(source_path)
touch(link_path)
self.assertRaises(RuntimeError, misc.create_symlink, source_path, link_path)
def existing_files_saved(self):
existing_file = os.path.join(self.destination_dir, 'foo.txt')
touch(existing_file)
new_dir = os.path.join(self.source_dir, 'bar')
os.makedirs(new_dir)
installdistributor.PuppetModuleInstallDistributor.\
_move_to_destination_directory(self.source_dir, self.destination_dir)
self.assertTrue(os.path.exists(existing_file))
def test_clean_ophaned_leaf(self):
"""
Test that an orphaned leaf is removed.
"""
leaf = os.path.join(self.publish_base, 'a', 'b', 'c', 'listing')
leaf_dir = os.path.dirname(leaf)
util.touch(leaf)
self.assertTrue(os.path.isfile(leaf))
migration.clean_simple_hosting_directories(leaf_dir, self.publish_base)
self.assertFalse(os.path.isdir(os.path.join(self.publish_base, 'a')))
def test_walk_does_not_recognize_non_leaf(self, mock_clean):
"""
Test that non orphaned leafs are not cleaned.
"""
non_orphan = os.path.join(self.publish_base, 'not', 'orphan', 'listing')
other_file = os.path.join(self.publish_base, 'not', 'orphan', 'notlisting')
util.touch(non_orphan)
util.touch(other_file)
migration.walk_and_clean_directories(self.publish_base)
self.assertEqual(mock_clean.call_count, 0)
def test_create_symlink_no_link_parent(self):
source_path = os.path.join(self.working_dir, 'source')
link_path = os.path.join(self.published_dir, 'foo/bar/baz/link')
touch(source_path)
self.assertFalse(os.path.exists(os.path.dirname(link_path)))
PublishStep._create_symlink(source_path, link_path)
self.assertTrue(os.path.exists(link_path))
def test_clean_with_concurrent_file_creation(self, mock_rmdir, mock_util):
"""
Clean directories when a dir cannot be removed during orphaned directory removal.
"""
mock_rmdir.side_effect = OSError()
listing_file_a = os.path.join(self.publish_base, 'a', 'listing')
updir = os.path.join(self.publish_base, 'a')
util.touch(listing_file_a)
old_symlink = os.path.join(self.publish_base, 'a', 'path_to_removed_symlink')
self.distributor.clean_simple_hosting_directories(old_symlink, self.publish_base)
mock_util.generate_listing_files.assert_called_once_with(updir, updir)
def test_create_symlink_no_link_parent_with_permissions(self, mock_makedirs, mock_symlink):
source_path = os.path.join(self.working_dir, 'source')
link_path = os.path.join(self.published_dir, 'foo/bar/baz/link')
touch(source_path)
self.assertFalse(os.path.exists(os.path.dirname(link_path)))
misc.create_symlink(source_path, link_path, directory_permissions=0700)
mock_makedirs.assert_called_once_with(os.path.dirname(link_path), mode=0700)
mock_symlink.assert_called_once_with(source_path, link_path)
def test_create_symlink_no_link_parent(self, mock_makedirs, mock_symlink):
source_path = os.path.join(self.working_dir, 'source')
link_path = os.path.join(self.published_dir, 'foo/bar/baz/link')
touch(source_path)
self.assertFalse(os.path.exists(os.path.dirname(link_path)))
misc.create_symlink(source_path, link_path)
mock_makedirs.assert_called_once_with(os.path.dirname(link_path), mode=0770)
mock_symlink.assert_called_once_with(source_path, link_path)
def test_clear_directory(self):
for file_name in ('one', 'two', 'three'):
touch(os.path.join(self.working_dir, file_name))
os.makedirs(os.path.join(self.working_dir, 'four'))
self.assertEqual(len(os.listdir(self.working_dir)), 4)
misc.clear_directory(self.working_dir, ['two'])
self.assertEqual(len(os.listdir(self.working_dir)), 1)
def test_clean_with_concurrent_file_creation(self, mock_rmdir, mock_util):
"""
Clean directories when a dir cannot be removed during orphaned directory removal.
"""
mock_rmdir.side_effect = OSError()
listing_file_a = os.path.join(self.publish_base, 'a', 'listing')
updir = os.path.join(self.publish_base, 'a')
util.touch(listing_file_a)
old_symlink = os.path.join(self.publish_base, 'a', 'path_to_removed_symlink')
self.distributor.clean_simple_hosting_directories(old_symlink, self.publish_base)
mock_util.generate_listing_files.assert_called_once_with(updir, updir)
def test_create_symlink_link_parent_bad_permissions(self):
source_path = os.path.join(self.working_dir, 'source')
link_path = os.path.join(self.published_dir, 'foo/bar/baz/link')
touch(source_path)
os.makedirs(os.path.dirname(link_path))
os.chmod(os.path.dirname(link_path), 0000)
self.assertRaises(OSError, PublishStep._create_symlink, source_path, link_path)
os.chmod(os.path.dirname(link_path), 0777)
def test_clear_directory(self):
for file_name in ("one", "two", "three"):
touch(os.path.join(self.working_dir, file_name))
os.makedirs(os.path.join(self.working_dir, "four"))
self.assertEqual(len(os.listdir(self.working_dir)), 4)
misc.clear_directory(self.working_dir, ["two"])
self.assertEqual(len(os.listdir(self.working_dir)), 1)
def test_process_unit(self):
step = publish_steps.PublishImagesStep()
fake_image_filename = 'fake-zero-byte-image.qcow2'
touch(os.path.join(self.content_directory, fake_image_filename))
unit = Mock(unit_key={'image_checksum': 'd41d8cd98f00b204e9800998ecf8427e'},
storage_path=os.path.join(self.content_directory, fake_image_filename))
step.get_working_dir = Mock(return_value=self.publish_directory)
step.process_unit(unit)
# verify symlink
expected_symlink = os.path.join(self.publish_directory, 'web', fake_image_filename)
self.assertTrue(os.path.exists(expected_symlink))
def test_create_symlink_no_link_parent_with_permissions(self, mock_makedirs, mock_symlink):
source_path = os.path.join(self.working_dir, "source")
link_path = os.path.join(self.published_dir, "foo/bar/baz/link")
touch(source_path)
self.assertFalse(os.path.exists(os.path.dirname(link_path)))
misc.create_symlink(source_path, link_path, directory_permissions=0700)
mock_makedirs.assert_called_once_with(os.path.dirname(link_path), mode=0700)
mock_symlink.assert_called_once_with(source_path, link_path)
def test_create_symlink(self):
source_path = os.path.join(self.working_dir, 'source')
link_path = os.path.join(self.published_dir, 'link')
touch(source_path)
self.assertFalse(os.path.exists(link_path))
PublishStep._create_symlink(source_path, link_path)
self.assertTrue(os.path.exists(link_path))
self.assertTrue(os.path.islink(link_path))
self.assertEqual(os.readlink(link_path), source_path)
def test_distributor_removed(self, mock_repo_dir):
mock_repo_dir.return_value = os.path.join(self.working_dir, 'repo')
os.makedirs(mock_repo_dir.return_value)
working_dir = os.path.join(self.working_dir, 'working')
repo = Mock(id='bar', working_dir=working_dir)
config = {}
touch(os.path.join(working_dir, 'bar.json'))
touch(os.path.join(mock_repo_dir.return_value, 'bar.tar'))
self.distributor.distributor_removed(repo, config)
self.assertEquals(0, len(os.listdir(mock_repo_dir.return_value)))
self.assertEquals(1, len(os.listdir(self.working_dir)))
def test_distributor_removed(self, mock_web, mock_master, mock_app, m_repo_objects):
m_repo_objects.get_repo_or_missing_resource.return_value = Mock(repo_id='bar')
mock_app.return_value = os.path.join(self.working_dir)
mock_web.return_value = os.path.join(self.working_dir, 'web')
mock_master.return_value = os.path.join(self.working_dir, 'master')
os.makedirs(mock_web.return_value)
os.makedirs(mock_master.return_value)
repo = Mock(id='bar')
config = {}
touch(os.path.join(self.working_dir, 'bar.json'))
self.distributor.distributor_removed(repo, config)
self.assertEquals(0, len(os.listdir(self.working_dir)))
def test_create_symlink_link_exists_and_is_correct(self):
new_source_path = os.path.join(self.working_dir, "new_source")
link_path = os.path.join(self.published_dir, "link")
touch(new_source_path)
os.symlink(new_source_path, link_path)
self.assertEqual(os.readlink(link_path), new_source_path)
misc.create_symlink(new_source_path, link_path)
self.assertEqual(os.readlink(link_path), new_source_path)
def test_process_main(self):
source_dir = os.path.join(self.working_directory, 'source')
os.makedirs(source_dir)
target_file = os.path.join(self.working_directory, 'target', 'target.tar')
step = publish_step.SaveTarFilePublishStep(source_dir, target_file)
touch(os.path.join(source_dir, 'foo.txt'))
step.process_main()
with contextlib.closing(tarfile.open(target_file)) as tar_file:
names = tar_file.getnames()
# the first item is either '' or '.' depending on if this is py2.7 or py2.6
self.assertEquals(names[1:], ['foo.txt'])
def test_create_symlink_link_exists_and_is_correct(self):
new_source_path = os.path.join(self.working_dir, 'new_source')
link_path = os.path.join(self.published_dir, 'link')
touch(new_source_path)
os.symlink(new_source_path, link_path)
self.assertEqual(os.readlink(link_path), new_source_path)
PublishStep._create_symlink(new_source_path, link_path)
self.assertEqual(os.readlink(link_path), new_source_path)
def test_process_main(self):
source_dir = os.path.join(self.working_directory, 'source')
os.makedirs(source_dir)
touch(os.path.join(source_dir, 'foo.txt'))
target_dir = os.path.join(self.working_directory, 'target')
target_file = os.path.join(target_dir, 'foo.txt')
step = publish_step.CopyDirectoryStep(source_dir, target_dir)
touch(os.path.join(source_dir, 'foo.txt'))
step.process_main()
self.assertTrue(os.path.exists(target_file))
def test_process_units(self):
step = publish_steps.PublishImagesStep()
step.parent = self.parent
step.redirect_context = Mock()
file_list = ['ancestry', 'layer', 'json']
for file_name in file_list:
touch(os.path.join(self.content_directory, file_name))
unit = Mock(unit_key={'image_id': 'foo_image'}, storage_path=self.content_directory)
step.get_working_dir = Mock(return_value=self.publish_directory)
step.process_unit(unit)
step.redirect_context.add_unit_metadata.assert_called_once_with(unit)
for file_name in file_list:
self.assertTrue(os.path.exists(os.path.join(self.publish_directory, 'web',
'foo_image', file_name)))
def test_distributor_removed(self, mock_web, mock_master, mock_app):
mock_app.return_value = os.path.join(self.working_dir)
mock_web.return_value = os.path.join(self.working_dir, 'web')
mock_master.return_value = os.path.join(self.working_dir, 'master')
working_dir = os.path.join(self.working_dir, 'working')
os.makedirs(mock_web.return_value)
os.makedirs(mock_master.return_value)
repo = Mock(id='bar', working_dir=working_dir)
config = {}
touch(os.path.join(self.working_dir, 'bar.json'))
self.distributor.distributor_removed(repo, config)
self.assertEquals(0, len(os.listdir(self.working_dir)))
def test_create_symlink_link_exists(self):
old_source_path = os.path.join(self.working_dir, "old_source")
new_source_path = os.path.join(self.working_dir, "new_source")
link_path = os.path.join(self.published_dir, "link")
touch(old_source_path)
touch(new_source_path)
os.symlink(old_source_path, link_path)
self.assertEqual(os.readlink(link_path), old_source_path)
link_path_with_slash = link_path + "/"
misc.create_symlink(new_source_path, link_path_with_slash)
self.assertEqual(os.readlink(link_path), new_source_path)
def test_create_symlink_link_exists(self):
old_source_path = os.path.join(self.working_dir, 'old_source')
new_source_path = os.path.join(self.working_dir, 'new_source')
link_path = os.path.join(self.published_dir, 'link')
touch(old_source_path)
touch(new_source_path)
os.symlink(old_source_path, link_path)
self.assertEqual(os.readlink(link_path), old_source_path)
link_path_with_slash = link_path + '/'
PublishStep._create_symlink(new_source_path, link_path_with_slash)
self.assertEqual(os.readlink(link_path), new_source_path)
def test_process_units(self):
step = publish_steps.PublishImagesStep()
step.parent = self.parent
step.redirect_context = Mock()
file_list = ['ancestry', 'layer', 'json']
for file_name in file_list:
touch(os.path.join(self.content_directory, file_name))
unit = Mock(unit_key={'image_id': 'foo_image'},
storage_path=self.content_directory)
step.get_working_dir = Mock(return_value=self.publish_directory)
step.process_unit(unit)
step.redirect_context.add_unit_metadata.assert_called_once_with(unit)
for file_name in file_list:
self.assertTrue(
os.path.exists(
os.path.join(self.publish_directory, 'web', 'foo_image',
file_name)))
def test_process_main(self, mock_get_unit):
mock_get_unit.return_value.storage_path = self.source_dir
content_dirs = ['objects', 'remote-cache', 'tmp', 'uncompressed-objects-cache']
for dir_name in content_dirs:
touch(os.path.join(self.source_dir, dir_name, 'foo'))
touch(os.path.join(self.source_dir, 'refs', 'foo'))
step = steps.PublishContentStep()
self.parent.add_child(step)
step.process_main()
for dir_name in content_dirs:
target_dir = os.path.join(self.target_dir, dir_name)
src_dir = os.path.join(self.source_dir, dir_name)
self.assertEquals(os.path.realpath(target_dir), src_dir)
self.assertFalse(os.path.exists(os.path.join(self.target_dir, 'refs')))
def test_clean_only_ophaned_leaf(self):
"""
Test partially shared path, only unshared portion of ophan path should be removed.
"""
listing_file_a = os.path.join(self.publish_base, 'a', 'listing')
listing_file_b = os.path.join(self.publish_base, 'a', 'b', 'listing')
listing_file_c = os.path.join(self.publish_base, 'a', 'b', 'c', 'listing')
non_orphan_listing = os.path.join(self.publish_base, 'a', 'other', 'listing')
non_orphan_file = os.path.join(self.publish_base, 'a', 'other', 'otherfile')
util.touch(listing_file_a)
util.touch(listing_file_b)
util.touch(listing_file_c)
util.touch(non_orphan_listing)
util.touch(non_orphan_file)
old_symlink = os.path.join(self.publish_base, 'a', 'b', 'c', 'path_to_removed_symlink')
self.distributor.clean_simple_hosting_directories(old_symlink, self.publish_base)
self.assertTrue(os.path.isdir(os.path.join(self.publish_base, 'a')))
self.assertFalse(os.path.isdir(os.path.join(self.publish_base, 'a', 'b')))
def test_clean_only_ophaned_leaf(self):
"""
Test partially shared path, only unshared portion of ophan path should be removed.
"""
listing_file_a = os.path.join(self.publish_base, 'a', 'listing')
listing_file_b = os.path.join(self.publish_base, 'a', 'b', 'listing')
listing_file_c = os.path.join(self.publish_base, 'a', 'b', 'c', 'listing')
non_orphan_listing = os.path.join(self.publish_base, 'a', 'other', 'listing')
non_orphan_file = os.path.join(self.publish_base, 'a', 'other', 'otherfile')
util.touch(listing_file_a)
util.touch(listing_file_b)
util.touch(listing_file_c)
util.touch(non_orphan_listing)
util.touch(non_orphan_file)
old_symlink = os.path.join(self.publish_base, 'a', 'b', 'c', 'path_to_removed_symlink')
self.distributor.clean_simple_hosting_directories(old_symlink, self.publish_base)
self.assertTrue(os.path.isdir(os.path.join(self.publish_base, 'a')))
self.assertFalse(os.path.isdir(os.path.join(self.publish_base, 'a', 'b')))
def test_pre_save_signal_directory_content(self, mock_config):
source_dir = os.path.join(self.working_dir, 'src')
target_dir = os.path.join(self.working_dir, 'target')
os.mkdir(source_dir)
os.mkdir(target_dir)
mock_config.config.get.return_value = target_dir
# create something in the source directory to test the copy
touch(os.path.join(source_dir, 'foo', 'bar'))
class ContentUnitHelper(model.ContentUnit):
unit_type_id = 'foo_unit'
foo = ContentUnitHelper()
foo.set_content(source_dir)
foo.id = 'abbcd'
model.ContentUnit.pre_save_signal(object(), foo)
full_path = os.path.join(target_dir, 'units', 'foo_unit', 'a', 'bb', 'abbcd', 'foo', 'bar')
self.assertTrue(os.path.exists(full_path))
def test_process_unit(self):
step = publish.PublishRpmAndDrpmStepIncremental()
self.publisher.add_child(step)
unit_key = {'name': 'foo', 'version': '1', 'release': '2', 'arch': 'flux'}
metadata = {'filename': 'bar.txt', 'repodata': 'baz', '_test': 'hidden'}
original_metadata = metadata.copy()
storage_path = os.path.join(self.working_dir, 'foo')
touch(storage_path)
test_unit = Unit('foo_type', unit_key, metadata, storage_path)
step.process_unit(test_unit)
original_metadata.pop('repodata')
original_metadata.pop('_test')
unit_file = os.path.join(self.working_dir, 'foo-1-2.flux.json')
self.assertTrue(os.path.exists(unit_file))
with open(unit_file) as file_handle:
loaded = json.load(file_handle)
compare_dict(loaded, {
'unit_key': unit_key, 'unit_metadata': original_metadata
})
def test_clean_ophaned_leaf(self):
"""
Test that an orphaned leaf is removed.
"""
listing_file_a = os.path.join(self.publish_base, 'a', 'listing')
listing_file_b = os.path.join(self.publish_base, 'a', 'b', 'listing')
listing_file_c = os.path.join(self.publish_base, 'a', 'b', 'c', 'listing')
util.touch(listing_file_a)
util.touch(listing_file_b)
util.touch(listing_file_c)
old_symlink = os.path.join(self.publish_base, 'a', 'b', 'c', 'path_to_removed_symlink')
self.distributor.clean_simple_hosting_directories(old_symlink, self.publish_base)
self.assertFalse(os.path.isdir(os.path.join(self.publish_base, 'a')))
def test_walk_recognizes_leaf(self, mock_clean):
"""
Test that an orphaned leaf is detected.
"""
leaf = os.path.join(self.publish_base, 'a', 'b', 'c', 'listing')
non_orphan = os.path.join(self.publish_base, 'not', 'leaf', 'listing')
other_file = os.path.join(self.publish_base, 'not', 'leaf',
'notlisting')
util.touch(leaf)
util.touch(non_orphan)
util.touch(other_file)
migration.walk_and_clean_directories(self.publish_base)
expected_leaf = os.path.join(self.publish_base, 'a', 'b', 'c')
mock_clean.assert_called_once_with(expected_leaf, self.publish_base)
