def test_use_nested_w_update(self):
settings.update({'foo': 0})
self.assertEquals(settings.get(), {'foo': 0})
with settings.use({'foo': 1}):
self.assertEquals(settings.get(), {'foo': 1})
self.assertEquals(settings._global_settings, {'foo': 0})
with settings.use({'foo': 2}):
self.assertEquals(settings.get(), {'foo': 2})
self.assertEquals(settings._global_settings, {'foo': 0})
self.assertEquals(settings.get(), {'foo': 1})
self.assertEquals(settings._global_settings, {'foo': 0})
self.assertEquals(settings.get(), {'foo': 0})
self.assertFalse(hasattr(settings.thread_local, 'settings'))
settings.update({'foo': 3})
self.assertEquals(settings.get(), {'foo': 3})
def test_static_large_obj_copy_and_segment_container(self):
with NamedTemporaryDirectory(change_dir=True) as tmp_d:
segment_size = 1048576
obj_size = segment_size * 4 + 100
self.create_dataset(tmp_d, 1, obj_size)
obj_path = stor.join(tmp_d,
self.get_dataset_obj_names(1)[0])
options = {'swift:upload': {'segment_size': segment_size}}
with settings.use(options):
obj_path.copy(self.test_container / 'large_object.txt')
# Verify there is a segment container and that it can be ignored when listing a dir
segment_container = Path(self.test_container.parent) / ('.segments_%s' % self.test_container.name) # noqa
containers = Path(self.test_container.parent).listdir(ignore_segment_containers=False)
self.assertTrue(segment_container in containers)
self.assertTrue(self.test_container in containers)
containers = Path(self.test_container.parent).listdir(ignore_segment_containers=True)
self.assertFalse(segment_container in containers)
self.assertTrue(self.test_container in containers)
# Verify there are five segments
objs = set(segment_container.list(condition=lambda results: len(results) == 5))
self.assertEquals(len(objs), 5)
# Copy back the large object and verify its contents
obj_path = Path(tmp_d) / 'large_object.txt'
Path(self.test_container / 'large_object.txt').copy(obj_path)
self.assertCorrectObjectContents(obj_path, self.get_dataset_obj_names(1)[0], obj_size)
def test_use_w_settings(self):
update_settings = {
'swift:upload': {
'skip_identical': True,
'object_threads': 30
},
'swift:download': {
'object_threads': 20
}
}
expected_settings = {
'swift:download': {
'container_threads': 10,
'object_threads': 20,
'shuffle': True,
'skip_identical': True
},
'swift:upload': {
'changed': False,
'checksum': True,
'leave_segments': True,
'object_threads': 30,
'segment_size': 1073741824,
'segment_threads': 10,
'skip_identical': True,
'use_slo': True
}
}
self.assertEquals(settings._global_settings, test_settings)
with settings.use(update_settings):
self.assertEquals(settings._global_settings, test_settings)
self.assertEquals(settings.get(), expected_settings)
self.assertEquals(settings._global_settings, test_settings)
self.assertEquals(settings.get(), test_settings)
def test_swift_destination(self, mock_upload):
source = '.'
dest = Path('swift://tenant/container')
options = {
'swift:upload': {
'object_threads': 30,
'segment_threads': 40
}
}
with settings.use(options):
utils.copytree(source, dest)
mock_upload.assert_called_once_with(dest, ['.'],
condition=None,
use_manifest=False,
headers=None)
def _use_contextmanager(self, value):
with settings.use({'foo': value}):
time.sleep(.01)
self.assertEquals(settings.get(), {'foo': value})
time.sleep(.01)
def test_use_update_w_error(self):
with settings.use({'foo': 1}):
with self.assertRaises(RuntimeError):
settings.update({'foo': 3})
def test_use_wo_settings(self):
self.assertEquals(settings._global_settings, test_settings)
with settings.use():
self.assertEquals(settings._global_settings, test_settings)
self.assertEquals(settings._global_settings, test_settings)
