def TearDownClass(cls):
"""Cleans up bucket and objects created by SetUpClass"""
if hasattr(cls, 'created_test_data'):
for test_obj_uri_str in cls.test_bucket0_obj_uri_strs:
test_util.test_storage_uri(test_obj_uri_str).delete_key()
for test_obj_uri_str in cls.test_bucket1_obj_uri_strs:
test_util.test_storage_uri(test_obj_uri_str).delete_key()
cls.test_bucket0_uri.delete_bucket()
cls.test_bucket1_uri.delete_bucket()
def TearDownClass(cls):
"""Cleans up bucket and objects created by SetUpClass"""
if hasattr(cls, 'created_test_data'):
for test_obj_uri_str in cls.test_bucket0_obj_uri_strs:
test_util.test_storage_uri(test_obj_uri_str).delete_key()
for test_obj_uri_str in cls.test_bucket1_obj_uri_strs:
test_util.test_storage_uri(test_obj_uri_str).delete_key()
cls.test_bucket0_uri.delete_bucket()
cls.test_bucket1_uri.delete_bucket()
def SetUpClass(cls):
"""Initializes test suite.
Creates a source bucket containing 3 objects;
a source directory containing a subdirectory and file;
and a destination bucket and directory.
"""
cls.uri_base_str = 'gs://gsutil_test_%s' % int(time.time())
# Use a designated tmpdir prefix to make it easy to find the end of
# the tmp path.
cls.tmpdir_prefix = 'tmp_gstest'
# Create the test buckets.
cls.src_bucket_uri = test_util.test_storage_uri('%s_src' %
cls.uri_base_str)
cls.dst_bucket_uri = test_util.test_storage_uri('%s_dst' %
cls.uri_base_str)
cls.src_bucket_uri.create_bucket()
cls.dst_bucket_uri.create_bucket()
# Create the test objects in src bucket.
cls.all_src_obj_uris = []
for i in range(3):
obj_uri = test_util.test_storage_uri('%sobj%d' %
(cls.src_bucket_uri, i))
cls.CreateEmptyObject(obj_uri)
cls.all_src_obj_uris.append(obj_uri)
# Create the test directories.
cls.src_dir_root = '%s%s' % (tempfile.mkdtemp(
prefix=cls.tmpdir_prefix), os.sep)
nested_subdir = '%sdir0%sdir1' % (cls.src_dir_root, os.sep)
os.makedirs(nested_subdir)
cls.dst_dir_root = '%s%s' % (tempfile.mkdtemp(
prefix=cls.tmpdir_prefix), os.sep)
# Create the test files in src directory.
cls.all_src_file_paths = []
cls.nested_child_file_paths = [
'f0', 'f1', 'f2.txt', 'dir0/dir1/nested'
]
cls.non_nested_file_names = ['f0', 'f1', 'f2.txt']
file_names = [
'f0', 'f1', 'f2.txt',
'dir0%sdir1%snested' % (os.sep, os.sep)
]
file_paths = ['%s%s' % (cls.src_dir_root, f) for f in file_names]
for file_path in file_paths:
f = open(file_path, 'w')
f.write('test data')
f.close()
cls.all_src_file_paths.append(file_path)
cls.created_test_data = True
def TestAttemptCopyingWithFileDirConflict(self):
"""Attempts to copy objects that cause a file/directory conflict"""
# Create objects with name conflicts (a/b and a). Use 'dst' bucket because
# it gets cleared after each test.
self.CreateEmptyObject(test_util.test_storage_uri(
'%sa/b' % self.dst_bucket_uri))
self.CreateEmptyObject(test_util.test_storage_uri(
'%sa' % self.dst_bucket_uri))
try:
command_inst.CopyObjsCommand([self.dst_bucket_uri.uri, self.dst_dir_root],
RECURSIVE, headers={})
self.fail('Did not get expected CommandException')
except CommandException, e:
self.assertNotEqual(e.reason.find(
'exists where a directory needs to be created'), -1)
def TestMatchingNonWildcardedUri(self):
"""Tests matching a single named file"""
exp_uri_strs = set(['file://%s/abcd' % self.test_dir])
uri = test_util.test_storage_uri('file://%s/abcd' % self.test_dir)
actual_uri_strs = set(
str(u) for u in test_util.test_wildcard_iterator(uri).IterUris())
self.assertEqual(exp_uri_strs, actual_uri_strs)
def TestAttemptCopyingWithFileDirConflict(self):
"""Attempts to copy objects that cause a file/directory conflict"""
# Create objects with name conflicts (a/b and a). Use 'dst' bucket because
# it gets cleared after each test.
self.CreateEmptyObject(
test_util.test_storage_uri('%sa/b' % self.dst_bucket_uri))
self.CreateEmptyObject(
test_util.test_storage_uri('%sa' % self.dst_bucket_uri))
try:
command_inst.CopyObjsCommand(
[self.dst_bucket_uri.uri, self.dst_dir_root], RECURSIVE)
self.fail('Did not get expected CommandException')
except CommandException, e:
self.assertNotEqual(
e.reason.find('exists where a directory needs to be created'),
-1)
def TestMatchingAllFiles(self):
"""Tests matching all files, based on wildcard"""
uri = test_util.test_storage_uri('file://%s/*' % self.test_dir)
actual_uri_strs = set(str(u) for u in
test_util.test_wildcard_iterator(uri).IterUris()
)
self.assertEqual(self.immed_child_uri_strs, actual_uri_strs)
def TestMatchingNonWildcardedUri(self):
"""Tests matching a single named file"""
exp_uri_strs = set(['file://%s/abcd' % self.test_dir])
uri = test_util.test_storage_uri('file://%s/abcd' % self.test_dir)
actual_uri_strs = set(
str(u) for u in test_util.test_wildcard_iterator(uri).IterUris())
self.assertEqual(exp_uri_strs, actual_uri_strs)
def SetUpClass(cls):
"""Initializes test suite.
Creates a source bucket containing 3 objects;
a source directory containing a subdirectory and file;
and a destination bucket and directory.
"""
cls.uri_base_str = 'gs://gsutil_test_%s' % int(time.time())
# Use a designated tmpdir prefix to make it easy to find the end of
# the tmp path.
cls.tmpdir_prefix = 'tmp_gstest'
# Create the test buckets.
cls.src_bucket_uri = test_util.test_storage_uri('%s_src' % cls.uri_base_str)
cls.dst_bucket_uri = test_util.test_storage_uri('%s_dst' % cls.uri_base_str)
cls.src_bucket_uri.create_bucket()
cls.dst_bucket_uri.create_bucket()
# Create the test objects in src bucket.
cls.all_src_obj_uris = []
for i in range(3):
obj_uri = test_util.test_storage_uri('%sobj%d' % (cls.src_bucket_uri, i))
cls.CreateEmptyObject(obj_uri)
cls.all_src_obj_uris.append(obj_uri)
# Create the test directories.
cls.src_dir_root = '%s%s' % (tempfile.mkdtemp(prefix=cls.tmpdir_prefix),
os.sep)
nested_subdir = '%sdir0%sdir1' % (cls.src_dir_root, os.sep)
os.makedirs(nested_subdir)
cls.dst_dir_root = '%s%s' % (tempfile.mkdtemp(prefix=cls.tmpdir_prefix),
os.sep)
# Create the test files in src directory.
cls.all_src_file_paths = []
cls.nested_child_file_paths = ['f0', 'f1', 'f2.txt', 'dir0/dir1/nested']
cls.non_nested_file_names = ['f0', 'f1', 'f2.txt']
file_names = ['f0', 'f1', 'f2.txt', 'dir0%sdir1%snested' % (os.sep, os.sep)]
file_paths = ['%s%s' % (cls.src_dir_root, f) for f in file_names]
for file_path in file_paths:
f = open(file_path, 'w')
f.write('test data')
f.close()
cls.all_src_file_paths.append(file_path)
cls.tmp_path = '%s%s' % (cls.src_dir_root, 'tmp0')
cls.created_test_data = True
def TestMatchingFilesIgnoringOtherRegexChars(self):
"""Tests ignoring non-wildcard regex chars (e.g., ^ and $)"""
exp_uri_strs = set(['file://%s/ade$' % self.test_dir])
uri = test_util.test_storage_uri('file://%s/ad*$' % self.test_dir)
actual_uri_strs = set(
str(u) for u in test_util.test_wildcard_iterator(uri).IterUris())
self.assertEqual(exp_uri_strs, actual_uri_strs)
def TestRecursiveDirectoryPlusFileWildcarding(self):
"""Tests recusive expansion of '**' directory plus '*' wildcard"""
uri = test_util.test_storage_uri('file://%s/**/*' % self.test_dir)
actual_uri_strs = set(str(u) for u in
test_util.test_wildcard_iterator(uri, ResultType.KEYS)
)
self.assertEqual(self.all_file_uri_strs, actual_uri_strs)
def TestMatchingAllFiles(self):
"""Tests matching all files, based on wildcard"""
uri = test_util.test_storage_uri('file://%s/*' % self.test_dir)
actual_uri_strs = set(str(u) for u in
test_util.test_wildcard_iterator(uri, ResultType.KEYS)
)
self.assertEqual(self.immed_child_uri_strs, actual_uri_strs)
def TestMatchingFilesIgnoringOtherRegexChars(self):
"""Tests ignoring non-wildcard regex chars (e.g., ^ and $)"""
exp_uri_strs = set(['file://%s/ade$' % self.test_dir])
uri = test_util.test_storage_uri('file://%s/ad*$' % self.test_dir)
actual_uri_strs = set(
str(u) for u in test_util.test_wildcard_iterator(uri).IterUris())
self.assertEqual(exp_uri_strs, actual_uri_strs)
def TestInvalidRecursiveDirectoryWildcard(self):
"""Tests that wildcard containing '***' raises exception"""
try:
uri = test_util.test_storage_uri('file://%s/***/abcd' % self.test_dir)
for unused_ in test_util.test_wildcard_iterator(uri).IterUris():
self.fail('Expected WildcardException not raised.')
except wildcard_iterator.WildcardException, e:
# Expected behavior.
self.assertTrue(str(e).find('more than 2 consecutive') != -1)
def __SetUpOneMockBucket(cls, bucket_num):
"""Creates a mock bucket containing 4 objects, including 1 nested.
Args:
bucket_num: Number for building bucket name.
Returns:
tuple: (bucket name, set of object URI strings).
"""
bucket_uri = test_util.test_storage_uri(
'%s_%s' % (cls.base_uri_str, bucket_num))
bucket_uri.create_bucket()
obj_uri_strs = set()
for obj_name in cls.all_obj_names:
obj_uri = test_util.test_storage_uri('%s%s' % (bucket_uri, obj_name))
key = obj_uri.new_key()
key.set_contents_from_string('')
obj_uri_strs.add(str(obj_uri))
return (bucket_uri, obj_uri_strs)
def TestInvalidRecursiveDirectoryWildcard(self):
"""Tests that wildcard containing '***' raises exception"""
try:
uri = test_util.test_storage_uri('file://%s/***/abcd' %
self.test_dir)
for unused_ in test_util.test_wildcard_iterator(uri).IterUris():
self.fail('Expected WildcardException not raised.')
except wildcard_iterator.WildcardException, e:
# Expected behavior.
self.assertTrue(str(e).find('more than 2 consecutive') != -1)
def TestMatchingFileSubset(self):
"""Tests matching a subset of files, based on wildcard"""
exp_uri_strs = set(
['file://%s/abcd' % self.test_dir, 'file://%s/abdd' % self.test_dir]
)
uri = test_util.test_storage_uri('file://%s/ab??' % self.test_dir)
actual_uri_strs = set(str(u) for u in
test_util.test_wildcard_iterator(uri).IterUris()
)
self.assertEqual(exp_uri_strs, actual_uri_strs)
def TestAttemptCopyingOverlappingSrcDstFile(self):
"""Attempts to an object atop itself"""
obj_uri = test_util.test_storage_uri('%sobj' % self.dst_bucket_uri)
self.CreateEmptyObject(obj_uri)
try:
self.RunCommand('cp', ['%s/f0' % self.src_dir_root,
'%s/f0' % self.src_dir_root])
self.fail('Did not get expected CommandException')
except CommandException, e:
self.assertNotEqual(e.reason.find('are the same file - abort'), -1)
def __SetUpOneMockBucket(cls, bucket_num):
"""Creates a mock bucket containing 4 objects, including 1 nested.
Args:
bucket_num: Number for building bucket name.
Returns:
tuple: (bucket name, set of object URI strings).
"""
bucket_uri = test_util.test_storage_uri('%s_%s' %
(cls.base_uri_str, bucket_num))
bucket_uri.create_bucket()
obj_uri_strs = set()
for obj_name in cls.all_obj_names:
obj_uri = test_util.test_storage_uri('%s%s' %
(bucket_uri, obj_name))
key = obj_uri.new_key()
key.set_contents_from_string('')
obj_uri_strs.add(str(obj_uri))
return (bucket_uri, obj_uri_strs)
def TestMatchingFileSubset(self):
"""Tests matching a subset of files, based on wildcard"""
exp_uri_strs = set([
'file://%s/abcd' % self.test_dir,
'file://%s/abdd' % self.test_dir
])
uri = test_util.test_storage_uri('file://%s/ab??' % self.test_dir)
actual_uri_strs = set(
str(u) for u in test_util.test_wildcard_iterator(uri).IterUris())
self.assertEqual(exp_uri_strs, actual_uri_strs)
def TestAttemptCopyingOverlappingSrcDst(self):
"""Attempts to an object atop itself"""
obj_uri = test_util.test_storage_uri('%sobj' % self.dst_bucket_uri)
self.CreateEmptyObject(obj_uri)
try:
command_inst.CopyObjsCommand(['%s*' % self.dst_bucket_uri.uri,
self.dst_bucket_uri.uri], headers={})
self.fail('Did not get expected CommandException')
except CommandException, e:
self.assertNotEqual(e.reason.find('are the same object - abort'), -1)
def TestExistingDirNoFileMatch(self):
"""Tests that wildcard raises exception when there's no match"""
try:
uri = test_util.test_storage_uri(
'file://%s/non_existent*' % self.test_dir)
for unused_ in test_util.test_wildcard_iterator(uri, ResultType.KEYS):
self.fail('Expected WildcardException not raised.')
except wildcard_iterator.WildcardException, e:
# Expected behavior.
self.assertTrue(str(e).find('No matches') != -1)
def TestAttemptCopyingOverlappingSrcDst(self):
"""Attempts to an object atop itself"""
obj_uri = test_util.test_storage_uri('%sobj' % self.dst_bucket_uri)
self.CreateEmptyObject(obj_uri)
try:
command_inst.CopyObjsCommand(
['%s*' % self.dst_bucket_uri.uri, self.dst_bucket_uri.uri])
self.fail('Did not get expected CommandException')
except CommandException, e:
self.assertNotEqual(e.reason.find('are the same object - abort'),
-1)
def TestAttemptCopyingWithDirFileConflict(self):
"""Attempts to copy an object that causes a directory/file conflict"""
# Create abc in dest dir.
os.mkdir('%sabc' % self.dst_dir_root)
# Create an object that conflicts with this dest subdir. Use 'dst' bucket
# as source because it gets cleared after each test.
obj_name = '%sabc' % self.dst_bucket_uri
self.CreateEmptyObject(test_util.test_storage_uri(obj_name))
try:
self.RunCommand('cp', [obj_name, self.dst_dir_root])
self.fail('Did not get expected CommandException')
except CommandException, e:
self.assertNotEqual(e.reason.find(
'where the file needs to be created'), -1)
def TestAttemptCopyingWithDirFileConflict(self):
"""Attempts to copy objects that cause a directory/file conflict"""
# Create subdir in dest dir.
os.mkdir('%ssubdir' % self.dst_dir_root)
# Create an object that conflicts with this dest subdir. Use 'dst' bucket
# because it gets cleared after each test.
self.CreateEmptyObject(test_util.test_storage_uri(
'%ssubdir' % self.dst_bucket_uri))
try:
command_inst.CopyObjsCommand([self.dst_bucket_uri.uri, self.dst_dir_root],
RECURSIVE, headers={})
self.fail('Did not get expected CommandException')
except CommandException, e:
self.assertNotEqual(e.reason.find(
'where the file needs to be created'), -1)
def TestAttemptCopyingWithDirFileConflict(self):
"""Attempts to copy objects that cause a directory/file conflict"""
# Create subdir in dest dir.
os.mkdir('%ssubdir' % self.dst_dir_root)
# Create an object that conflicts with this dest subdir. Use 'dst' bucket
# because it gets cleared after each test.
self.CreateEmptyObject(
test_util.test_storage_uri('%ssubdir' % self.dst_bucket_uri))
try:
command_inst.CopyObjsCommand(
[self.dst_bucket_uri.uri, self.dst_dir_root], RECURSIVE)
self.fail('Did not get expected CommandException')
except CommandException, e:
self.assertNotEqual(
e.reason.find('where the file needs to be created'), -1)
def TestWildcardMoveWithinBucket(self):
"""Attempts to move using src wildcard that overlaps dest object.
We want to ensure that this doesn't stomp the result data. See the
comment starting with "Expand wildcards before" in MoveObjsCommand
for details.
"""
# Create a single object; use 'dst' bucket because it gets cleared after
# each test.
self.CreateEmptyObject(
test_util.test_storage_uri('%sold' % self.dst_bucket_uri))
self.command_runner.RunNamedCommand('mv', ['%s*' % self.dst_bucket_uri.uri,
'%snew' % self.dst_bucket_uri.uri])
actual = list(
test_util.test_wildcard_iterator('%s*' % self.dst_bucket_uri.uri))
self.assertEqual(1, len(actual))
self.assertEqual('new', actual[0].object_name)
def TestWildcardMoveWithinBucket(self):
"""Attempts to move using src wildcard that overlaps dest object.
We want to ensure that this doesn't stomp the result data. See the
comment starting with "Expand wildcards before" in MoveObjsCommand
for details.
"""
# Create a single object; use 'dst' bucket because it gets cleared after
# each test.
self.CreateEmptyObject(
test_util.test_storage_uri('%sold' % self.dst_bucket_uri))
command_inst.MoveObjsCommand([
'%s*' % self.dst_bucket_uri.uri,
'%snew' % self.dst_bucket_uri.uri
])
actual = list(
test_util.test_wildcard_iterator('%s*' % self.dst_bucket_uri.uri))
self.assertEqual(1, len(actual))
self.assertEqual('new', actual[0].object_name)
def SetUpClass(cls):
"""Initializes test suite.
Creates a source bucket containing 9 objects, 3 of which live under a
subdir, 3 of which lives under a nested subdir; a source directory
containing a subdirectory and file; and a destination bucket and directory.
"""
cls.uri_base_str = 'gs://gsutil_test_%s' % int(time.time())
# Use a designated tmpdir prefix to make it easy to find the end of
# the tmp path.
cls.tmpdir_prefix = 'tmp_gstest'
# Create the test buckets.
cls.src_bucket_uri = test_util.test_storage_uri('%s_src' % cls.uri_base_str)
cls.dst_bucket_uri = test_util.test_storage_uri('%s_dst' % cls.uri_base_str)
cls.src_bucket_uri.create_bucket()
# Define the src and dest bucket subdir paths. Note that they exclude
# a slash on the end so we can test handling of bucket subdirs specified
# both with and without terminating slashes.
cls.src_bucket_subdir_uri = test_util.test_storage_uri(
'%s_src/src_subdir' % cls.uri_base_str)
cls.src_bucket_subdir_uri_wildcard = test_util.test_storage_uri(
'%s_src/src_sub*' % cls.uri_base_str)
cls.dst_bucket_subdir_uri = test_util.test_storage_uri(
'%s_dst/dst_subdir' % cls.uri_base_str)
cls.dst_bucket_uri.create_bucket()
# Create the test objects in the src bucket.
cls.all_src_obj_uris = []
cls.all_src_top_level_obj_uris = []
cls.all_src_subdir_obj_uris = []
cls.all_src_subdir_and_below_obj_uris = []
for i in range(3):
obj_uri = test_util.test_storage_uri('%sobj%d' % (cls.src_bucket_uri, i))
cls.CreateEmptyObject(obj_uri)
cls.all_src_obj_uris.append(obj_uri)
cls.all_src_top_level_obj_uris.append(obj_uri)
# Subdir objects
for i in range(4, 6):
obj_uri = test_util.test_storage_uri(
'%s/obj%d' % (cls.src_bucket_subdir_uri, i))
cls.CreateEmptyObject(obj_uri)
cls.all_src_obj_uris.append(obj_uri)
cls.all_src_subdir_obj_uris.append(obj_uri)
cls.all_src_subdir_and_below_obj_uris.append(obj_uri)
# Nested subdir objects
for i in range(7, 9):
obj_uri = test_util.test_storage_uri(
'%s/nested/obj%d' % (cls.src_bucket_subdir_uri, i))
cls.CreateEmptyObject(obj_uri)
cls.all_src_obj_uris.append(obj_uri)
cls.all_src_subdir_and_below_obj_uris.append(obj_uri)
# Create the test directories.
cls.src_dir_root = '%s%s' % (tempfile.mkdtemp(prefix=cls.tmpdir_prefix),
os.sep)
nested_subdir = '%sdir0%sdir1' % (cls.src_dir_root, os.sep)
os.makedirs(nested_subdir)
cls.dst_dir_root = '%s%s' % (tempfile.mkdtemp(prefix=cls.tmpdir_prefix),
os.sep)
# Create the test files in the src directory.
cls.all_src_file_paths = []
cls.nested_child_file_paths = ['f0', 'f1', 'f2.txt', 'dir0/dir1/nested']
cls.non_nested_file_names = ['f0', 'f1', 'f2.txt']
file_names = ['f0', 'f1', 'f2.txt', 'dir0%sdir1%snested' % (os.sep, os.sep)]
file_paths = ['%s%s' % (cls.src_dir_root, f) for f in file_names]
for file_path in file_paths:
f = open(file_path, 'w')
f.write('test data')
f.close()
cls.all_src_file_paths.append(file_path)
cls.tmp_path = '%s%s' % (cls.src_dir_root, 'tmp0')
cls.created_test_data = True
def TestRecursiveDirectoryOnlyWildcarding(self):
"""Tests recusive expansion of directory-only '**' wildcard"""
uri = test_util.test_storage_uri('file://%s/**' % self.test_dir)
actual_uri_strs = set(
str(u) for u in test_util.test_wildcard_iterator(uri).IterUris())
self.assertEqual(self.all_file_uri_strs, actual_uri_strs)
def TestExistingDirNoFileMatch(self):
"""Tests that wildcard returns empty iterator when there's no match"""
uri = test_util.test_storage_uri('file://%s/non_existent*' %
self.test_dir)
res = list(test_util.test_wildcard_iterator(uri).IterUris())
self.assertEqual(0, len(res))
def TestRecursiveDirectoryOnlyWildcarding(self):
"""Tests recusive expansion of directory-only '**' wildcard"""
uri = test_util.test_storage_uri('file://%s/**' % self.test_dir)
actual_uri_strs = set(
str(u) for u in test_util.test_wildcard_iterator(uri).IterUris())
self.assertEqual(self.all_file_uri_strs, actual_uri_strs)
def TestExistingDirNoFileMatch(self):
"""Tests that wildcard returns empty iterator when there's no match"""
uri = test_util.test_storage_uri(
'file://%s/non_existent*' % self.test_dir)
res = list(test_util.test_wildcard_iterator(uri).IterUris())
self.assertEqual(0, len(res))
