def test_s3_cp_relative_paths(self):
s3.cp(self.local_file, 'test-DELETEME', force=True)
self.assertTrue(os.path.exists(os.path.join(os.getcwd(), 'test-DELETEME')))
s3.cp('test-DELETEME', self.remote_file("local"), force=True)
self.assert_s3_exists(self.remote_file("local"))
s3.rm('test-DELETEME')
self.assertFalse(os.path.exists(os.path.join(os.getcwd(), 'test-DELETEME')))
def test_s3_ls(self):
files = ["foo", "bar.baz", "quack/foo.foo"]
for f in files:
s3.cp(self.local_file, self.remote_file(f))
self.assertEqual(set(s3.ls(self.s3_test_location)), set(map(lambda x: self.s3_path+x, files)))
self.assertEqual(set(s3.ls(self.s3_test_location, return_full_urls=True)), set([self.remote_file(x) for x in files]))
self.assertEqual(set(s3.ls(self.s3_test_location, shallow=True)), set([self.s3_path+x for x in ['foo', 'bar.baz', 'quack/']]))
def test_s3_cp_HOME_paths(self):
s3.cp(self.local_file, '~/test-DELETEME', force=True)
self.assertTrue(os.path.exists(os.path.expanduser('~/test-DELETEME')))
s3.cp('~/test-DELETEME', self.remote_file("local"), force=True)
self.assert_s3_exists(self.remote_file("local"))
s3.rm('~/test-DELETEME')
self.assertFalse(os.path.exists(os.path.expanduser('~/test-DELETEME')))
def test_s3_cp_download_versioned_raise_key_not_found_with_unknown_version_id(self):
from baiji.exceptions import KeyNotFound
unknown_version_id = '5elgojhtA8BGJerqfbciN78eU74SJ9mX'
# test raise KeyNotFound with unknown versionId
with self.assertRaises(KeyNotFound):
s3.cp(self.existing_versioned_remote_file, os.path.join(self.tmp_dir, 'DL', 'TEST.foo'), version_id=unknown_version_id)
def test_s3_cp_download_versioned_raise_invalid_version_id_with_bad_version_id(self):
from baiji.exceptions import InvalidVersionID
invalid_version_id = '1111'
# test raise S3ResponseError with invalid versionId
with self.assertRaises(InvalidVersionID):
s3.cp(self.existing_versioned_remote_file, os.path.join(self.tmp_dir, 'DL', 'TEST.foo'), version_id=invalid_version_id)
def test_that_invalidating_tree_removes_child_timestamps(self):
import uuid
path = 'test_sc/{}'.format(uuid.uuid4())
filenames = ['{}/test_sample_{}.txt'.format(path, i) for i in range(3)]
for filename in filenames:
remote_file = 's3://{}/{}'.format(self.bucket, filename)
s3.cp(self.temp_file, remote_file)
self.cache(filename)
s3.rm(remote_file)
for filename in filenames:
timestamp_file = os.path.join(
self.cache_dir, '.timestamps', self.bucket, filename)
self.assertTrue(os.path.exists(timestamp_file))
cache_file = os.path.join(
self.cache_dir, self.bucket, filename)
self.assertTrue(os.path.exists(cache_file))
self.cache.invalidate(path)
for filename in filenames:
timestamp_file = os.path.join(
self.cache_dir, '.timestamps', self.bucket, filename)
self.assertFalse(os.path.exists(timestamp_file))
cache_file = os.path.join(
self.cache_dir, self.bucket, filename)
self.assertTrue(os.path.exists(cache_file))
def test_s3_glob_exclude(self):
files = ['pose_T.obj', 'pose_A.obj', 'pose_Scan.obj']
for f in files:
s3.cp(self.local_file, self.remote_file(f))
glob_results = list(s3.glob(self.s3_test_location, 'pose_[!T].obj'))
self.assertEqual(1, len(glob_results))
self.assertEqual(set(glob_results), set([self.remote_file('pose_A.obj')]))
def test_s3_cp_download_versioned_success_with_valid_version_id(self):
version_id = s3.info(self.existing_versioned_remote_file)['version_id']
s3.cp(self.existing_versioned_remote_file,
os.path.join(self.tmp_dir, 'DL', 'TEST.foo'),
version_id=version_id)
self.assertTrue(
os.path.exists(os.path.join(self.tmp_dir, 'DL', 'TEST.foo')))
def test_that_invalidating_tree_removes_child_timestamps(self):
import uuid
path = 'test_sc/{}'.format(uuid.uuid4())
filenames = ['{}/test_sample_{}.txt'.format(path, i) for i in range(3)]
for filename in filenames:
remote_file = 's3://{}/{}'.format(self.bucket, filename)
s3.cp(self.temp_file, remote_file)
self.cache(filename)
s3.rm(remote_file)
for filename in filenames:
timestamp_file = os.path.join(
self.cache_dir, '.timestamps', self.bucket, filename)
self.assertTrue(os.path.exists(timestamp_file))
cache_file = os.path.join(
self.cache_dir, self.bucket, filename)
self.assertTrue(os.path.exists(cache_file))
self.cache.invalidate(path)
for filename in filenames:
timestamp_file = os.path.join(
self.cache_dir, '.timestamps', self.bucket, filename)
self.assertFalse(os.path.exists(timestamp_file))
cache_file = os.path.join(
self.cache_dir, self.bucket, filename)
self.assertTrue(os.path.exists(cache_file))
def test_s3_cp_download_lookup_recover_in_one_retry(
self, ensure_integrity_mock):
from baiji.exceptions import KeyNotFound
ensure_integrity_mock.side_effect = [
KeyNotFound('key not found'), None
]
s3.cp(self.existing_remote_file, self.tmp_dir, force=True)
def test_s3_cp_download_corrupted_recover_in_one_retry(
self, ensure_integrity_mock):
from baiji.exceptions import get_transient_error_class
ensure_integrity_mock.side_effect = [
get_transient_error_class()('etag does not match'), None
]
s3.cp(self.existing_remote_file, self.tmp_dir, force=True)
def test_s3_cp_HOME_paths(self):
s3.cp(self.local_file, '~/test-DELETEME', force=True)
self.assertTrue(os.path.exists(os.path.expanduser('~/test-DELETEME')))
s3.cp('~/test-DELETEME', self.remote_file("local"), force=True)
self.assert_s3_exists(self.remote_file("local"))
s3.rm('~/test-DELETEME')
self.assertFalse(os.path.exists(os.path.expanduser('~/test-DELETEME')))
def test_s3_cp_policy(self):
# test policy for file -> s3
s3.cp(self.local_file,
self.remote_file("public.md"),
policy='public-read')
self.assert_is_public(self.remote_file("public.md"), is_public=True)
s3.cp(self.local_file,
self.remote_file("private.md"),
policy='bucket-owner-read')
self.assert_is_public(self.remote_file("private.md"), is_public=False)
# test policy for s3 -> s3
s3.cp(self.remote_file("private.md"),
self.remote_file("made_public_on_copy.md"),
policy='public-read')
self.assert_is_public(self.remote_file("made_public_on_copy.md"),
is_public=True)
s3.cp(self.remote_file("private.md"),
self.remote_file("left_private_on_copy.md"))
self.assert_is_public(self.remote_file("left_private_on_copy.md"),
is_public=False)
with self.assertRaises(ValueError):
s3.cp(self.remote_file("private.md"),
os.path.join(self.tmp_dir, 'NoCanDo.txt'),
policy='public-read')
def test_s3_cp_errors_if_source_is_missing(self):
# remote
with self.assertRaisesRegexp(s3.KeyNotFound, "Error copying"):
s3.cp(self.remote_file("definitely_not_there.foo"), self.tmp_dir)
# local
with self.assertRaisesRegexp(s3.KeyNotFound, "Error copying"):
s3.cp(os.path.join(self.tmp_dir, "definitely_not_there.foo"), self.tmp_dir)
def test_s3_glob_match_single_wildcard(self):
files = ['a.obj', 'b.obj', 'a.ply', 'a.object']
for f in files:
s3.cp(self.local_file, self.remote_file(f))
glob_results = list(s3.glob(self.s3_test_location, '*.obj'))
self.assertEqual(2, len(glob_results))
self.assertEqual(set(glob_results), set([self.remote_file(f) for f in ['a.obj', 'b.obj']]))
def main(self):
from baiji import s3
args = self._parse_args()
vc = self._create_vc(manifest_path=args.manifest, bucket=args.bucket)
if args.command == 'add':
vc.add(args.path, args.file, verbose=True)
if args.command == 'update':
vc.update(
args.path,
args.file,
major=args.major,
minor=args.minor,
patch=args.patch,
verbose=True)
if args.command == 'versions':
for v in vc.versions_available(args.path):
print v
if args.command == 'sync':
print 'sync to {}'.format(args.destination)
vc.sync(args.destination)
if args.command == 'ls':
print '\n'.join(sorted(vc.manifest_files))
if args.command == 'ls-remote':
print '\n'.join(sorted(vc.ls_remote()))
if args.command == 'get':
f = vc(args.path, version=args.version)
print 'copying {} version {} to {}'.format(
args.path,
vc.manifest_version(args.path),
args.destination)
s3.cp(f, args.destination)
if args.command == 'path':
print vc(args.path, version=args.version)
if args.command == 'open':
import subprocess
subprocess.call(['open', vc(args.path, version=args.version)])
if args.command == 'path-remote':
print vc.uri(args.path, version=args.version)
if args.command == 'cat':
import shutil
import sys
f = vc(args.path, version=args.version)
shutil.copyfileobj(open(f, 'rb'), sys.stdout)
# On success, exit with status code of 0.
return 0
def main(self):
from baiji import s3
args = self._parse_args()
vc = self._create_vc(manifest_path=args.manifest, bucket=args.bucket)
if args.command == 'add':
vc.add(args.path, args.file, verbose=True)
if args.command == 'update':
vc.update(
args.path,
args.file,
major=args.major,
minor=args.minor,
patch=args.patch,
verbose=True)
if args.command == 'versions':
for v in vc.versions_available(args.path):
print(v)
if args.command == 'sync':
print('sync to {}'.format(args.destination))
vc.sync(args.destination)
if args.command == 'ls':
print('\n'.join(sorted(vc.manifest_files)))
if args.command == 'ls-remote':
print('\n'.join(sorted(vc.ls_remote())))
if args.command == 'get':
f = vc(args.path, version=args.version)
print('copying {} version {} to {}'.format(
args.path,
vc.manifest_version(args.path),
args.destination))
s3.cp(f, args.destination)
if args.command == 'path':
print(vc(args.path, version=args.version))
if args.command == 'open':
import subprocess
subprocess.call(['open', vc(args.path, version=args.version)])
if args.command == 'path-remote':
print(vc.uri(args.path, version=args.version))
if args.command == 'cat':
import shutil
import sys
f = vc(args.path, version=args.version)
shutil.copyfileobj(open(f, 'rb'), sys.stdout)
# On success, exit with status code of 0.
return 0
def main(self, key):
if self.fix:
etag = s3.etag(key)
if "-" in etag:
s3.cp(key, key, force=True)
print "etag updated from {} to {}".format(etag, s3.etag(key))
else:
print s3.etag(key)
def test_s3_cp_errors_without_permissions(self):
from baiji.util.shutillib import mkdir_p
locked_dir = os.path.join(self.tmp_dir, 'locked')
mkdir_p(locked_dir)
os.chmod(locked_dir, 666)
with self.assertRaises(s3.S3Exception):
s3.cp(self.local_file, os.path.join(locked_dir, 'nope.txt'))
self.assertFalse(os.path.exists(os.path.join(locked_dir, 'nope.txt')))
def test_s3_cp_errors_if_source_is_missing(self):
# remote
with self.assertRaisesRegexp(s3.KeyNotFound, "Error copying"):
s3.cp(self.remote_file("definitely_not_there.foo"), self.tmp_dir)
# local
with self.assertRaisesRegexp(s3.KeyNotFound, "Error copying"):
s3.cp(os.path.join(self.tmp_dir, "definitely_not_there.foo"),
self.tmp_dir)
def main(self, key):
if self.fix:
etag = s3.etag(key)
if "-" in etag:
s3.cp(key, key, force=True)
print "etag updated from {} to {}".format(etag, s3.etag(key))
else:
print s3.etag(key)
def sync(self, destination):
for f in self.manifest_files:
target = s3.path.join(destination, f[1:])
print 'Copying {} version {} to {}'.format(
f,
self.manifest_version(f),
target)
s3.cp(self(f), target, force=True)
def test_s3_glob_exclude(self):
files = ['pose_T.obj', 'pose_A.obj', 'pose_Scan.obj']
for f in files:
s3.cp(self.local_file, self.remote_file(f))
glob_results = list(s3.glob(self.s3_test_location, 'pose_[!T].obj'))
self.assertEqual(1, len(glob_results))
self.assertEqual(set(glob_results),
set([self.remote_file('pose_A.obj')]))
def test_s3_cp_errors_without_permissions(self):
from baiji.util.shutillib import mkdir_p
locked_dir = os.path.join(self.tmp_dir, 'locked')
mkdir_p(locked_dir)
os.chmod(locked_dir, 666)
with self.assertRaises(s3.S3Exception):
s3.cp(self.local_file, os.path.join(locked_dir, 'nope.txt'))
self.assertFalse(os.path.exists(os.path.join(locked_dir, 'nope.txt')))
def sync(self, destination):
for f in self.manifest_files:
target = s3.path.join(destination, f[1:])
print('Copying {} version {} to {}'.format(
f,
self.manifest_version(f),
target))
s3.cp(self(f), target, force=True)
def test_s3_cp_download_corrupted_raise_transient_error_after_retried_once(self, ensure_integrity_mock):
from baiji.exceptions import get_transient_error_class
ensure_integrity_mock.side_effect = get_transient_error_class()('etag does not match')
with self.assertRaises(get_transient_error_class()):
s3.cp(self.existing_remote_file, self.tmp_dir, force=True)
def test_encrypt_in_place(self):
s3.cp(self.local_file,
self.remote_file("to_encrypt.txt"),
encrypt=False) # just make sure there's something to copy
self.assertFalse(
s3.info(self.remote_file("to_encrypt.txt"))['encrypted'])
s3.encrypt_at_rest(self.remote_file("to_encrypt.txt"))
self.assertTrue(
s3.info(self.remote_file("to_encrypt.txt"))['encrypted'])
def existing_remote_file(self):
'''
In some tests it is convenient to have a file already on s3;
in some others we need it not to be there (e.g. for clarity in the s3.ls test)
'''
uri = self.remote_file("FOO/A_preexisting_file.md")
if not s3.exists(uri):
s3.cp(self.local_file, uri)
return uri
def test_downloading_a_directory_without_slash_that_is_also_a_file(self):
s3.touch(self.remote_file("foo"))
s3.touch(self.remote_file("foo/theres_a_file_in_here.txt"))
# This should work, so that you have some way to download a legit file that's also a dir
s3.cp(self.remote_file("foo"), os.path.join(self.tmp_dir, "foo"))
self.assertEqual(len(os.listdir(self.tmp_dir)), 1)
# But this will fail, as there's already a file in place so we can't make the dir "foo"
with self.assertRaises(s3.S3Exception):
s3.cp(self.remote_file("foo/theres_a_file_in_here.txt"), os.path.join(self.tmp_dir, "foo/"))
def test_downloads_from_s3_are_atomic_under_truncation(self, download_mock):
from baiji.exceptions import get_transient_error_class
def write_fake_truncated_file(fp, **kwargs): # just capturing whatever is thrown at us: pylint: disable=unused-argument
fp.write("12345")
download_mock.side_effect = write_fake_truncated_file
# Now when the call to download the file is made, the etags won't match
with self.assertRaises(get_transient_error_class()):
s3.cp(self.existing_remote_file, os.path.join(self.tmp_dir, 'truncated.foo'), validate=True)
self.assertFalse(os.path.exists(os.path.join(self.tmp_dir, 'truncated.foo')))
def test_s3_rm(self):
for path in [os.path.join(self.tmp_dir, 'foo'), self.remote_file("foo")]:
s3.cp(self.local_file, path)
self.assert_s3_exists(path)
s3.rm(path)
self.assert_s3_does_not_exist(path)
with self.assertRaises(s3.KeyNotFound):
s3.rm(path)
self.assertRaises(s3.InvalidSchemeException, s3.rm, ("http://example.com/foo"))
def test_s3_glob_match_single_wildcard(self):
files = ['a.obj', 'b.obj', 'a.ply', 'a.object']
for f in files:
s3.cp(self.local_file, self.remote_file(f))
glob_results = list(s3.glob(self.s3_test_location, '*.obj'))
self.assertEqual(2, len(glob_results))
self.assertEqual(
set(glob_results),
set([self.remote_file(f) for f in ['a.obj', 'b.obj']]))
def existing_remote_file(self):
'''
In some tests it is convenient to have a file already on s3;
in some others we need it not to be there (e.g. for clarity in the s3.ls test)
'''
uri = self.remote_file("FOO/A_preexisting_file.md")
if not s3.exists(uri):
s3.cp(self.local_file, uri)
return uri
def test_downloads_from_s3_are_atomic_under_exceptions(self, download_mock):
download_mock.side_effect = ValueError()
# Now when the call to download the file is made, an exception will be thrown.
# ideally, we'd throw it "in" boto via a mock, but we really want to test that
# the file doesn't get written, so let's go ahead and let boto do the download
# and then throw the exception in the validation
with self.assertRaises(ValueError):
s3.cp(self.existing_remote_file, os.path.join(self.tmp_dir, 'erroneous.foo'), validate=True)
self.assertFalse(os.path.exists(os.path.join(self.tmp_dir, 'erroneous.foo')))
def test_s3_cp_relative_paths(self):
s3.cp(self.local_file, 'test-DELETEME', force=True)
self.assertTrue(
os.path.exists(os.path.join(os.getcwd(), 'test-DELETEME')))
s3.cp('test-DELETEME', self.remote_file("local"), force=True)
self.assert_s3_exists(self.remote_file("local"))
s3.rm('test-DELETEME')
self.assertFalse(
os.path.exists(os.path.join(os.getcwd(), 'test-DELETEME')))
def test_downloading_a_directory_without_slash_that_is_also_a_file(self):
s3.touch(self.remote_file("foo"))
s3.touch(self.remote_file("foo/theres_a_file_in_here.txt"))
# This should work, so that you have some way to download a legit file that's also a dir
s3.cp(self.remote_file("foo"), os.path.join(self.tmp_dir, "foo"))
self.assertEqual(len(os.listdir(self.tmp_dir)), 1)
# But this will fail, as there's already a file in place so we can't make the dir "foo"
with self.assertRaises(s3.S3Exception):
s3.cp(self.remote_file("foo/theres_a_file_in_here.txt"),
os.path.join(self.tmp_dir, "foo/"))
def test_s3_glob_match_multiple_wildcards(self):
files = ['body_1_pose_T.obj', 'body_1_pose_Fun.obj', 'body_2_pose_T.obj', 'body_02_pose_T.obj']
for f in files:
s3.cp(self.local_file, self.remote_file(f))
glob_results = list(s3.glob(self.s3_test_location, 'body_?_pose_*.obj'))
self.assertEqual(3, len(glob_results))
self.assertEqual(
set(glob_results),
set([self.remote_file(f) for f in ['body_1_pose_T.obj', 'body_1_pose_Fun.obj', 'body_2_pose_T.obj']])
)
def test_etag_on_multipart_upload(self):
five_mb = 5 * 1024 * 1024
big_local_file = create_random_temporary_file(int(five_mb + 1024))
self.assertGreater(os.path.getsize(big_local_file), five_mb)
remote_multipart = self.remote_file("TestEtag/multipart.md")
s3.cp(big_local_file, remote_multipart, max_size=five_mb)
self.assertIn("-", s3.etag(remote_multipart))
self.assertNotIn("-", s3.etag(big_local_file))
self.assertTrue(s3.etag_matches(big_local_file, s3.etag(remote_multipart)))
os.remove(big_local_file)
def test_s3_cp_download_corrupted_raise_transient_error_after_retried_once(
self, ensure_integrity_mock):
from baiji.exceptions import get_transient_error_class
ensure_integrity_mock.side_effect = get_transient_error_class()(
'etag does not match')
with self.assertRaises(get_transient_error_class()):
s3.cp(self.existing_remote_file, self.tmp_dir, force=True)
def test_copy(self):
s3.cp(self.local_file,
self.remote_file("unencrypted.txt"),
encrypt=False) # just make sure there's something to copy
self.assertFalse(
s3.info(self.remote_file("unencrypted.txt"))['encrypted'])
s3.cp(self.remote_file("unencrypted.txt"),
self.remote_file("encrypted.txt"))
self.assertTrue(
s3.info(self.remote_file("encrypted.txt"))['encrypted'])
def test_s3_cp_download_versioned_raise_key_not_found_with_unknown_version_id(
self):
from baiji.exceptions import KeyNotFound
unknown_version_id = '5elgojhtA8BGJerqfbciN78eU74SJ9mX'
# test raise KeyNotFound with unknown versionId
with self.assertRaises(KeyNotFound):
s3.cp(self.existing_versioned_remote_file,
os.path.join(self.tmp_dir, 'DL', 'TEST.foo'),
version_id=unknown_version_id)
def test_s3_cp_download_versioned_raise_invalid_version_id_with_bad_version_id(
self):
from baiji.exceptions import InvalidVersionID
invalid_version_id = '1111'
# test raise S3ResponseError with invalid versionId
with self.assertRaises(InvalidVersionID):
s3.cp(self.existing_versioned_remote_file,
os.path.join(self.tmp_dir, 'DL', 'TEST.foo'),
version_id=invalid_version_id)
def test_upload(self):
s3.cp(self.local_file,
self.remote_file("unencrypted.txt"),
encrypt=False)
self.assertFalse(
s3.info(self.remote_file("unencrypted.txt"))['encrypted'])
s3.cp(self.local_file,
self.remote_file("encrypted.txt")) # default now to encrypt
self.assertTrue(
s3.info(self.remote_file("encrypted.txt"))['encrypted'])
def test_s3_cp_download(self):
s3.cp(self.existing_remote_file,
os.path.join(self.tmp_dir, 'DL', 'TEST.foo'))
self.assertTrue(
os.path.exists(os.path.join(self.tmp_dir, 'DL', 'TEST.foo')))
s3.cp(self.existing_remote_file, os.path.join(self.tmp_dir, 'DL'))
self.assertTrue(
os.path.exists(
os.path.join(self.tmp_dir, 'DL',
s3.path.basename(self.existing_remote_file))))
def test_s3_with_double_slashes_in_key(self):
'''
boto has a nasty behavior by default where it collapses `//` to `/` in keys
'''
s3.cp(self.local_file,
self.remote_file('double//slashes//bork//boto.foo'))
self.assertEqual([self.remote_file('double//slashes//bork//boto.foo')],
list(
s3.ls(self.remote_file(''),
return_full_urls=True)))
def test_s3_cp_gzip(self):
s3.cp(self.local_file, self.remote_file("big.md"))
s3.cp(self.local_file, self.remote_file("small.md"), gzip=True)
self.assertNotEqual(
s3.info(self.remote_file("big.md"))['content_encoding'], 'gzip')
self.assertEqual(
s3.info(self.remote_file("small.md"))['content_encoding'], 'gzip')
self.assertLess(
s3.info(self.remote_file("small.md"))['size'],
s3.info(self.remote_file("big.md"))['size'])
def test_etag_on_multipart_upload(self):
five_mb = 5 * 1024 * 1024
big_local_file = create_random_temporary_file(int(five_mb + 1024))
self.assertGreater(os.path.getsize(big_local_file), five_mb)
remote_multipart = self.remote_file("TestEtag/multipart.md")
s3.cp(big_local_file, remote_multipart, max_size=five_mb)
self.assertIn("-", s3.etag(remote_multipart))
self.assertNotIn("-", s3.etag(big_local_file))
self.assertTrue(
s3.etag_matches(big_local_file, s3.etag(remote_multipart)))
os.remove(big_local_file)
def test_s3_ls(self):
files = ["foo", "bar.baz", "quack/foo.foo"]
for f in files:
s3.cp(self.local_file, self.remote_file(f))
self.assertEqual(set(s3.ls(self.s3_test_location)),
set(map(lambda x: self.s3_path + x, files)))
self.assertEqual(
set(s3.ls(self.s3_test_location, return_full_urls=True)),
set([self.remote_file(x) for x in files]))
self.assertEqual(
set(s3.ls(self.s3_test_location, shallow=True)),
set([self.s3_path + x for x in ['foo', 'bar.baz', 'quack/']]))
def test_downloads_from_s3_are_atomic_under_exceptions(
self, download_mock):
download_mock.side_effect = ValueError()
# Now when the call to download the file is made, an exception will be thrown.
# ideally, we'd throw it "in" boto via a mock, but we really want to test that
# the file doesn't get written, so let's go ahead and let boto do the download
# and then throw the exception in the validation
with self.assertRaises(ValueError):
s3.cp(self.existing_remote_file,
os.path.join(self.tmp_dir, 'erroneous.foo'),
validate=True)
self.assertFalse(
os.path.exists(os.path.join(self.tmp_dir, 'erroneous.foo')))
def test_s3_rm(self):
for path in [
os.path.join(self.tmp_dir, 'foo'),
self.remote_file("foo")
]:
s3.cp(self.local_file, path)
self.assert_s3_exists(path)
s3.rm(path)
self.assert_s3_does_not_exist(path)
with self.assertRaises(s3.KeyNotFound):
s3.rm(path)
self.assertRaises(s3.InvalidSchemeException, s3.rm,
("http://example.com/foo"))
def existing_versioned_remote_file(self):
# use a hardcoded path for test versioned file on S3
# to avoid bookkeeping
# the current test won't make versioned copies of the file
# the remote object will be either deleted (which will be overwritten later)
# or download to local
uri = 's3://{}/FOO/A_preexisting_file.md'.format(VERSIONED_TEST_BUCKET)
if not s3.exists(uri):
s3.cp(self.local_file, uri)
return uri
def existing_versioned_remote_file(self):
# use a hardcoded path for test versioned file on S3
# to avoid bookkeeping
# the current test won't make versioned copies of the file
# the remote object will be either deleted (which will be overwritten later)
# or download to local
uri = 's3://baiji-test-versioned/FOO/A_preexisting_file.md'
if not s3.exists(uri):
s3.cp(self.local_file, uri)
return uri
def add(self, path, local_file, version=None, verbose=False):
path = self.normalize_path(path)
if self.is_versioned(path):
raise ValueError('{} is already versioned; did you mean vc.update?'.format(path))
if version is None:
version = '1.0.0'
else:
version = self.normalize_version_number(version)
if not self.version_number_is_valid(version):
raise ValueError('invalid version {}, always use versions of the form N.N.N'.format(version))
s3.cp(local_file, self.uri(path, version), progress=verbose)
self.update_manifest(path, version)
def test_does_check_after_timeout(self):
import time
self.cache(self.filename)
s3.cp(self.get_test_file_path(), self.remote_file, force=True)
time.sleep(2)
with mock.patch('baiji.s3.cp') as mock_cp:
mock_cp.return_value = True
self.cache(self.filename)
mock_cp.assert_called_with(
self.remote_file, self.local_file,
progress=False, force=True, validate=True)
def setUp(self):
import uuid
from bltest.random_data import create_random_temporary_file
super(TestAssetCache, self).setUp()
self.filename = 'test_sc/{}/test_sample.txt'.format(uuid.uuid4())
self.local_file = os.path.join(self.cache_dir, self.bucket, self.filename)
self.timestamp_file = os.path.join(
self.cache_dir, '.timestamps', self.bucket, self.filename)
self.remote_file = 's3://{}/{}'.format(self.bucket, self.filename)
self.temp_file = create_random_temporary_file()
s3.cp(self.temp_file, self.remote_file)
def test_s3_open_write_remote_file_with_context_manager(self):
remote_file_name = self.remote_file("write_test_1")
local_file_name = os.path.join(self.tmp_dir, "write_test_1")
self.assert_s3_does_not_exist(remote_file_name)
with s3.open(remote_file_name, 'w') as f:
tempname = f.name
f.write(self.truth)
self.assertFalse(os.path.exists(tempname))
self.assert_s3_exists(remote_file_name)
# download and confirm that it contains the correct contents
s3.cp(remote_file_name, local_file_name)
with open(local_file_name) as f:
self.assertEqual(self.truth, f.read())
def main(self, src, dst):
kwargs = {
'force': self.force,
'progress': self.progress,
'policy': self.policy,
'preserve_acl': self.preserve_acl,
'encoding': self.encoding,
'encrypt': self.encrypt,
'gzip': self.gzip,
'skip': self.skip,
'version_id': self.version_id,
}
if self.recursive or self.recursive_parallel:
s3.cp_r(src, dst, parallel=self.recursive_parallel, **kwargs)
else:
s3.cp(src, dst, **kwargs)
def test_s3_cp_policy(self):
# test policy for file -> s3
s3.cp(self.local_file, self.remote_file("public.md"), policy='public-read')
self.assert_is_public(self.remote_file("public.md"), is_public=True)
s3.cp(self.local_file, self.remote_file("private.md"), policy='bucket-owner-read')
self.assert_is_public(self.remote_file("private.md"), is_public=False)
# test policy for s3 -> s3
s3.cp(self.remote_file("private.md"), self.remote_file("made_public_on_copy.md"), policy='public-read')
self.assert_is_public(self.remote_file("made_public_on_copy.md"), is_public=True)
s3.cp(self.remote_file("private.md"), self.remote_file("left_private_on_copy.md"))
self.assert_is_public(self.remote_file("left_private_on_copy.md"), is_public=False)
with self.assertRaises(ValueError):
s3.cp(self.remote_file("private.md"), os.path.join(self.tmp_dir, 'NoCanDo.txt'), policy='public-read')
def test_s3_cp_preserve_acl(self):
s3.cp(self.local_file, self.remote_file("also_public.md"), policy='public-read')
s3.cp(self.remote_file("also_public.md"), self.remote_file("still_public.md"), preserve_acl=True)
self.assert_is_public(self.remote_file("also_public.md"), is_public=True)
s3.cp(self.remote_file("also_public.md"), self.remote_file("no_longer_public.md"))
self.assert_is_public(self.remote_file("no_longer_public.md"), is_public=False)
with self.assertRaises(ValueError):
s3.cp(self.remote_file("also_public.md"), os.path.join(self.tmp_dir, 'NoCanDo.txt'), preserve_acl=True)
def test_s3_cp_content_encoding(self):
s3.cp(self.local_file, self.remote_file("encoded.md"), encoding='gzip')
self.assertEqual(s3.info(self.remote_file("encoded.md"))['content_encoding'], 'gzip')
s3.cp(self.local_file, self.remote_file("notencoded.md")) # just make sure gzip isn't the default ;)
self.assertNotEqual(s3.info(self.remote_file("notencoded.md"))['content_encoding'], 'gzip')
s3.cp(self.remote_file("encoded.md"), self.remote_file("still_encoded.md"))
self.assertEqual(s3.info(self.remote_file("still_encoded.md"))['content_encoding'], 'gzip')
s3.cp(self.remote_file("notencoded.md"), self.remote_file("now_encoded.md"), encoding='gzip')
self.assertEqual(s3.info(self.remote_file("now_encoded.md"))['content_encoding'], 'gzip')
