def test_rmtree(self):
with NamedTemporaryDirectory(change_dir=True) as tmp_d:
# Make a couple empty test files and nested files
tmp_d = Path(tmp_d)
os.mkdir(tmp_d / 'my_dir')
open(tmp_d / 'my_dir' / 'dir_file1', 'w').close()
open(tmp_d / 'my_dir' / 'dir_file2', 'w').close()
open(tmp_d / 'base_file1', 'w').close()
open(tmp_d / 'base_file2', 'w').close()
stor.copytree(
'.',
self.test_container,
use_manifest=True)
swift_dir = self.test_container / 'my_dir'
self.assertEquals(len(swift_dir.list()), 2)
swift_dir.rmtree()
self.assertEquals(len(swift_dir.list()), 0)
base_contents = self.test_container.list()
self.assertTrue((self.test_container / 'base_file1') in base_contents)
self.assertTrue((self.test_container / 'base_file1') in base_contents)
self.test_container.rmtree()
# TODO figure out a better way to test that the container no longer exists.
with self.assertRaises(swift.NotFoundError):
# Replication may have not happened yet for container deletion. Keep
# listing in intervals until a NotFoundError is thrown
for i in (0, 1, 3):
time.sleep(i)
self.test_container.list()
def test_copytree_to_from_dir_w_manifest(self):
num_test_objs = 10
test_obj_size = 100
with NamedTemporaryDirectory(change_dir=True) as tmp_d:
self.create_dataset(tmp_d, num_test_objs, test_obj_size)
# Make a nested file and an empty directory for testing purposes
tmp_d = Path(tmp_d)
os.mkdir(tmp_d / 'my_dir')
open(tmp_d / 'my_dir' / 'empty_file', 'w').close()
os.mkdir(tmp_d / 'my_dir' / 'empty_dir')
stor.copytree('.', self.test_dir, use_manifest=True)
# Validate the contents of the manifest file
manifest_contents = utils.get_data_manifest_contents(self.test_dir)
expected_contents = self.get_dataset_obj_names(num_test_objs)
expected_contents.extend(
['my_dir/empty_file', 'my_dir/empty_dir/'])
expected_contents = [Path('test') / c for c in expected_contents]
self.assertEquals(set(manifest_contents), set(expected_contents))
with NamedTemporaryDirectory(change_dir=True) as tmp_d:
# Download the results successfully
Path(self.test_dir).copytree('test', use_manifest=True)
# Now delete one of the objects from s3. A second download
# will fail with a condition error
Path(self.test_dir / 'my_dir' / 'empty_dir/').remove()
with self.assertRaises(exceptions.ConditionNotMetError):
Path(self.test_dir).copytree('test',
use_manifest=True,
num_retries=0)
def test_copytree_w_headers(self):
with NamedTemporaryDirectory(change_dir=True) as tmp_d:
open(tmp_d / 'test_obj', 'w').close()
stor.copytree('.',
self.test_container,
headers=['X-Delete-After:1000'])
obj = stor.join(self.test_container, 'test_obj')
stat_results = obj.stat()
self.assertTrue('x-delete-at' in stat_results['headers'])
def test_non_empty_directory_errors(self):
example_dir = stor.join(self.test_dir, 'example')
assert not example_dir.exists()
other_dir = stor.join(self.test_dir, 'otherdir')
self.create_dataset(other_dir, 1, 1)
self.create_dataset(example_dir, 1, 1)
example_dir.makedirs_p()
with self.assertRaisesRegexp(FileExistsError, '.*File exists'):
try:
stor.copytree(other_dir, example_dir)
except FileExistsError as e:
assert e.errno == 17
raise
def test_copytree_to_from_dir(self):
num_test_objs = 10
test_obj_size = 100
with NamedTemporaryDirectory(change_dir=True) as tmp_d:
self.create_dataset(tmp_d, num_test_objs, test_obj_size)
stor.copytree('.', self.test_dir)
time.sleep(10) # for uploaded files to close
with NamedTemporaryDirectory(change_dir=True) as tmp_d:
self.test_dir.copytree('test')
# Verify contents of all downloaded test objects
for which_obj in self.get_dataset_obj_names(num_test_objs):
obj_path = Path('test') / which_obj
self.assertCorrectObjectContents(obj_path, which_obj,
test_obj_size)
def test_copytree_to_from_dir(self):
num_test_objs = 10
test_obj_size = 100
with NamedTemporaryDirectory(change_dir=True) as tmp_d:
self.create_dataset(tmp_d, num_test_objs, test_obj_size)
stor.copytree('.', self.test_dir)
with NamedTemporaryDirectory(change_dir=True) as tmp_d:
self.test_dir.copytree(
'test',
condition=lambda results: len(results) == num_test_objs)
# Verify contents of all downloaded test objects
for which_obj in self.get_dataset_obj_names(num_test_objs):
obj_path = Path('test') / which_obj
self.assertCorrectObjectContents(obj_path, which_obj,
test_obj_size)
