def test_s3_cp_relative_paths(self): s3.cp(self.local_file, 'test-DELETEME', force=True) self.assertTrue(os.path.exists(os.path.join(os.getcwd(), 'test-DELETEME'))) s3.cp('test-DELETEME', self.remote_file("local"), force=True) self.assert_s3_exists(self.remote_file("local")) s3.rm('test-DELETEME') self.assertFalse(os.path.exists(os.path.join(os.getcwd(), 'test-DELETEME')))
def test_s3_ls(self): files = ["foo", "bar.baz", "quack/foo.foo"] for f in files: s3.cp(self.local_file, self.remote_file(f)) self.assertEqual(set(s3.ls(self.s3_test_location)), set(map(lambda x: self.s3_path+x, files))) self.assertEqual(set(s3.ls(self.s3_test_location, return_full_urls=True)), set([self.remote_file(x) for x in files])) self.assertEqual(set(s3.ls(self.s3_test_location, shallow=True)), set([self.s3_path+x for x in ['foo', 'bar.baz', 'quack/']]))
def test_s3_cp_HOME_paths(self): s3.cp(self.local_file, '~/test-DELETEME', force=True) self.assertTrue(os.path.exists(os.path.expanduser('~/test-DELETEME'))) s3.cp('~/test-DELETEME', self.remote_file("local"), force=True) self.assert_s3_exists(self.remote_file("local")) s3.rm('~/test-DELETEME') self.assertFalse(os.path.exists(os.path.expanduser('~/test-DELETEME')))
def test_s3_cp_download_versioned_raise_key_not_found_with_unknown_version_id(self): from baiji.exceptions import KeyNotFound unknown_version_id = '5elgojhtA8BGJerqfbciN78eU74SJ9mX' # test raise KeyNotFound with unknown versionId with self.assertRaises(KeyNotFound): s3.cp(self.existing_versioned_remote_file, os.path.join(self.tmp_dir, 'DL', 'TEST.foo'), version_id=unknown_version_id)
def test_s3_cp_download_versioned_raise_invalid_version_id_with_bad_version_id(self): from baiji.exceptions import InvalidVersionID invalid_version_id = '1111' # test raise S3ResponseError with invalid versionId with self.assertRaises(InvalidVersionID): s3.cp(self.existing_versioned_remote_file, os.path.join(self.tmp_dir, 'DL', 'TEST.foo'), version_id=invalid_version_id)
def test_that_invalidating_tree_removes_child_timestamps(self): import uuid path = 'test_sc/{}'.format(uuid.uuid4()) filenames = ['{}/test_sample_{}.txt'.format(path, i) for i in range(3)] for filename in filenames: remote_file = 's3://{}/{}'.format(self.bucket, filename) s3.cp(self.temp_file, remote_file) self.cache(filename) s3.rm(remote_file) for filename in filenames: timestamp_file = os.path.join( self.cache_dir, '.timestamps', self.bucket, filename) self.assertTrue(os.path.exists(timestamp_file)) cache_file = os.path.join( self.cache_dir, self.bucket, filename) self.assertTrue(os.path.exists(cache_file)) self.cache.invalidate(path) for filename in filenames: timestamp_file = os.path.join( self.cache_dir, '.timestamps', self.bucket, filename) self.assertFalse(os.path.exists(timestamp_file)) cache_file = os.path.join( self.cache_dir, self.bucket, filename) self.assertTrue(os.path.exists(cache_file))
def test_s3_glob_exclude(self): files = ['pose_T.obj', 'pose_A.obj', 'pose_Scan.obj'] for f in files: s3.cp(self.local_file, self.remote_file(f)) glob_results = list(s3.glob(self.s3_test_location, 'pose_[!T].obj')) self.assertEqual(1, len(glob_results)) self.assertEqual(set(glob_results), set([self.remote_file('pose_A.obj')]))
def test_s3_cp_download_versioned_success_with_valid_version_id(self): version_id = s3.info(self.existing_versioned_remote_file)['version_id'] s3.cp(self.existing_versioned_remote_file, os.path.join(self.tmp_dir, 'DL', 'TEST.foo'), version_id=version_id) self.assertTrue( os.path.exists(os.path.join(self.tmp_dir, 'DL', 'TEST.foo')))
def test_s3_cp_download_lookup_recover_in_one_retry( self, ensure_integrity_mock): from baiji.exceptions import KeyNotFound ensure_integrity_mock.side_effect = [ KeyNotFound('key not found'), None ] s3.cp(self.existing_remote_file, self.tmp_dir, force=True)
def test_s3_cp_download_corrupted_recover_in_one_retry( self, ensure_integrity_mock): from baiji.exceptions import get_transient_error_class ensure_integrity_mock.side_effect = [ get_transient_error_class()('etag does not match'), None ] s3.cp(self.existing_remote_file, self.tmp_dir, force=True)
def test_s3_cp_policy(self): # test policy for file -> s3 s3.cp(self.local_file, self.remote_file("public.md"), policy='public-read') self.assert_is_public(self.remote_file("public.md"), is_public=True) s3.cp(self.local_file, self.remote_file("private.md"), policy='bucket-owner-read') self.assert_is_public(self.remote_file("private.md"), is_public=False) # test policy for s3 -> s3 s3.cp(self.remote_file("private.md"), self.remote_file("made_public_on_copy.md"), policy='public-read') self.assert_is_public(self.remote_file("made_public_on_copy.md"), is_public=True) s3.cp(self.remote_file("private.md"), self.remote_file("left_private_on_copy.md")) self.assert_is_public(self.remote_file("left_private_on_copy.md"), is_public=False) with self.assertRaises(ValueError): s3.cp(self.remote_file("private.md"), os.path.join(self.tmp_dir, 'NoCanDo.txt'), policy='public-read')
def test_s3_cp_errors_if_source_is_missing(self): # remote with self.assertRaisesRegexp(s3.KeyNotFound, "Error copying"): s3.cp(self.remote_file("definitely_not_there.foo"), self.tmp_dir) # local with self.assertRaisesRegexp(s3.KeyNotFound, "Error copying"): s3.cp(os.path.join(self.tmp_dir, "definitely_not_there.foo"), self.tmp_dir)
def test_s3_glob_match_single_wildcard(self): files = ['a.obj', 'b.obj', 'a.ply', 'a.object'] for f in files: s3.cp(self.local_file, self.remote_file(f)) glob_results = list(s3.glob(self.s3_test_location, '*.obj')) self.assertEqual(2, len(glob_results)) self.assertEqual(set(glob_results), set([self.remote_file(f) for f in ['a.obj', 'b.obj']]))
def main(self): from baiji import s3 args = self._parse_args() vc = self._create_vc(manifest_path=args.manifest, bucket=args.bucket) if args.command == 'add': vc.add(args.path, args.file, verbose=True) if args.command == 'update': vc.update( args.path, args.file, major=args.major, minor=args.minor, patch=args.patch, verbose=True) if args.command == 'versions': for v in vc.versions_available(args.path): print v if args.command == 'sync': print 'sync to {}'.format(args.destination) vc.sync(args.destination) if args.command == 'ls': print '\n'.join(sorted(vc.manifest_files)) if args.command == 'ls-remote': print '\n'.join(sorted(vc.ls_remote())) if args.command == 'get': f = vc(args.path, version=args.version) print 'copying {} version {} to {}'.format( args.path, vc.manifest_version(args.path), args.destination) s3.cp(f, args.destination) if args.command == 'path': print vc(args.path, version=args.version) if args.command == 'open': import subprocess subprocess.call(['open', vc(args.path, version=args.version)]) if args.command == 'path-remote': print vc.uri(args.path, version=args.version) if args.command == 'cat': import shutil import sys f = vc(args.path, version=args.version) shutil.copyfileobj(open(f, 'rb'), sys.stdout) # On success, exit with status code of 0. return 0
def main(self): from baiji import s3 args = self._parse_args() vc = self._create_vc(manifest_path=args.manifest, bucket=args.bucket) if args.command == 'add': vc.add(args.path, args.file, verbose=True) if args.command == 'update': vc.update( args.path, args.file, major=args.major, minor=args.minor, patch=args.patch, verbose=True) if args.command == 'versions': for v in vc.versions_available(args.path): print(v) if args.command == 'sync': print('sync to {}'.format(args.destination)) vc.sync(args.destination) if args.command == 'ls': print('\n'.join(sorted(vc.manifest_files))) if args.command == 'ls-remote': print('\n'.join(sorted(vc.ls_remote()))) if args.command == 'get': f = vc(args.path, version=args.version) print('copying {} version {} to {}'.format( args.path, vc.manifest_version(args.path), args.destination)) s3.cp(f, args.destination) if args.command == 'path': print(vc(args.path, version=args.version)) if args.command == 'open': import subprocess subprocess.call(['open', vc(args.path, version=args.version)]) if args.command == 'path-remote': print(vc.uri(args.path, version=args.version)) if args.command == 'cat': import shutil import sys f = vc(args.path, version=args.version) shutil.copyfileobj(open(f, 'rb'), sys.stdout) # On success, exit with status code of 0. return 0
def main(self, key): if self.fix: etag = s3.etag(key) if "-" in etag: s3.cp(key, key, force=True) print "etag updated from {} to {}".format(etag, s3.etag(key)) else: print s3.etag(key)
def test_s3_cp_errors_without_permissions(self): from baiji.util.shutillib import mkdir_p locked_dir = os.path.join(self.tmp_dir, 'locked') mkdir_p(locked_dir) os.chmod(locked_dir, 666) with self.assertRaises(s3.S3Exception): s3.cp(self.local_file, os.path.join(locked_dir, 'nope.txt')) self.assertFalse(os.path.exists(os.path.join(locked_dir, 'nope.txt')))
def sync(self, destination): for f in self.manifest_files: target = s3.path.join(destination, f[1:]) print 'Copying {} version {} to {}'.format( f, self.manifest_version(f), target) s3.cp(self(f), target, force=True)
def sync(self, destination): for f in self.manifest_files: target = s3.path.join(destination, f[1:]) print('Copying {} version {} to {}'.format( f, self.manifest_version(f), target)) s3.cp(self(f), target, force=True)
def test_s3_cp_download_corrupted_raise_transient_error_after_retried_once(self, ensure_integrity_mock): from baiji.exceptions import get_transient_error_class ensure_integrity_mock.side_effect = get_transient_error_class()('etag does not match') with self.assertRaises(get_transient_error_class()): s3.cp(self.existing_remote_file, self.tmp_dir, force=True)
def test_encrypt_in_place(self): s3.cp(self.local_file, self.remote_file("to_encrypt.txt"), encrypt=False) # just make sure there's something to copy self.assertFalse( s3.info(self.remote_file("to_encrypt.txt"))['encrypted']) s3.encrypt_at_rest(self.remote_file("to_encrypt.txt")) self.assertTrue( s3.info(self.remote_file("to_encrypt.txt"))['encrypted'])
def existing_remote_file(self): ''' In some tests it is convenient to have a file already on s3; in some others we need it not to be there (e.g. for clarity in the s3.ls test) ''' uri = self.remote_file("FOO/A_preexisting_file.md") if not s3.exists(uri): s3.cp(self.local_file, uri) return uri
def test_downloading_a_directory_without_slash_that_is_also_a_file(self): s3.touch(self.remote_file("foo")) s3.touch(self.remote_file("foo/theres_a_file_in_here.txt")) # This should work, so that you have some way to download a legit file that's also a dir s3.cp(self.remote_file("foo"), os.path.join(self.tmp_dir, "foo")) self.assertEqual(len(os.listdir(self.tmp_dir)), 1) # But this will fail, as there's already a file in place so we can't make the dir "foo" with self.assertRaises(s3.S3Exception): s3.cp(self.remote_file("foo/theres_a_file_in_here.txt"), os.path.join(self.tmp_dir, "foo/"))
def test_downloads_from_s3_are_atomic_under_truncation(self, download_mock): from baiji.exceptions import get_transient_error_class def write_fake_truncated_file(fp, **kwargs): # just capturing whatever is thrown at us: pylint: disable=unused-argument fp.write("12345") download_mock.side_effect = write_fake_truncated_file # Now when the call to download the file is made, the etags won't match with self.assertRaises(get_transient_error_class()): s3.cp(self.existing_remote_file, os.path.join(self.tmp_dir, 'truncated.foo'), validate=True) self.assertFalse(os.path.exists(os.path.join(self.tmp_dir, 'truncated.foo')))
def test_s3_rm(self): for path in [os.path.join(self.tmp_dir, 'foo'), self.remote_file("foo")]: s3.cp(self.local_file, path) self.assert_s3_exists(path) s3.rm(path) self.assert_s3_does_not_exist(path) with self.assertRaises(s3.KeyNotFound): s3.rm(path) self.assertRaises(s3.InvalidSchemeException, s3.rm, ("http://example.com/foo"))
def test_s3_glob_match_single_wildcard(self): files = ['a.obj', 'b.obj', 'a.ply', 'a.object'] for f in files: s3.cp(self.local_file, self.remote_file(f)) glob_results = list(s3.glob(self.s3_test_location, '*.obj')) self.assertEqual(2, len(glob_results)) self.assertEqual( set(glob_results), set([self.remote_file(f) for f in ['a.obj', 'b.obj']]))
def test_downloads_from_s3_are_atomic_under_exceptions(self, download_mock): download_mock.side_effect = ValueError() # Now when the call to download the file is made, an exception will be thrown. # ideally, we'd throw it "in" boto via a mock, but we really want to test that # the file doesn't get written, so let's go ahead and let boto do the download # and then throw the exception in the validation with self.assertRaises(ValueError): s3.cp(self.existing_remote_file, os.path.join(self.tmp_dir, 'erroneous.foo'), validate=True) self.assertFalse(os.path.exists(os.path.join(self.tmp_dir, 'erroneous.foo')))
def test_s3_cp_relative_paths(self): s3.cp(self.local_file, 'test-DELETEME', force=True) self.assertTrue( os.path.exists(os.path.join(os.getcwd(), 'test-DELETEME'))) s3.cp('test-DELETEME', self.remote_file("local"), force=True) self.assert_s3_exists(self.remote_file("local")) s3.rm('test-DELETEME') self.assertFalse( os.path.exists(os.path.join(os.getcwd(), 'test-DELETEME')))
def test_s3_glob_match_multiple_wildcards(self): files = ['body_1_pose_T.obj', 'body_1_pose_Fun.obj', 'body_2_pose_T.obj', 'body_02_pose_T.obj'] for f in files: s3.cp(self.local_file, self.remote_file(f)) glob_results = list(s3.glob(self.s3_test_location, 'body_?_pose_*.obj')) self.assertEqual(3, len(glob_results)) self.assertEqual( set(glob_results), set([self.remote_file(f) for f in ['body_1_pose_T.obj', 'body_1_pose_Fun.obj', 'body_2_pose_T.obj']]) )
def test_etag_on_multipart_upload(self): five_mb = 5 * 1024 * 1024 big_local_file = create_random_temporary_file(int(five_mb + 1024)) self.assertGreater(os.path.getsize(big_local_file), five_mb) remote_multipart = self.remote_file("TestEtag/multipart.md") s3.cp(big_local_file, remote_multipart, max_size=five_mb) self.assertIn("-", s3.etag(remote_multipart)) self.assertNotIn("-", s3.etag(big_local_file)) self.assertTrue(s3.etag_matches(big_local_file, s3.etag(remote_multipart))) os.remove(big_local_file)
def test_s3_cp_download_corrupted_raise_transient_error_after_retried_once( self, ensure_integrity_mock): from baiji.exceptions import get_transient_error_class ensure_integrity_mock.side_effect = get_transient_error_class()( 'etag does not match') with self.assertRaises(get_transient_error_class()): s3.cp(self.existing_remote_file, self.tmp_dir, force=True)
def test_copy(self): s3.cp(self.local_file, self.remote_file("unencrypted.txt"), encrypt=False) # just make sure there's something to copy self.assertFalse( s3.info(self.remote_file("unencrypted.txt"))['encrypted']) s3.cp(self.remote_file("unencrypted.txt"), self.remote_file("encrypted.txt")) self.assertTrue( s3.info(self.remote_file("encrypted.txt"))['encrypted'])
def test_s3_cp_download_versioned_raise_key_not_found_with_unknown_version_id( self): from baiji.exceptions import KeyNotFound unknown_version_id = '5elgojhtA8BGJerqfbciN78eU74SJ9mX' # test raise KeyNotFound with unknown versionId with self.assertRaises(KeyNotFound): s3.cp(self.existing_versioned_remote_file, os.path.join(self.tmp_dir, 'DL', 'TEST.foo'), version_id=unknown_version_id)
def test_s3_cp_download_versioned_raise_invalid_version_id_with_bad_version_id( self): from baiji.exceptions import InvalidVersionID invalid_version_id = '1111' # test raise S3ResponseError with invalid versionId with self.assertRaises(InvalidVersionID): s3.cp(self.existing_versioned_remote_file, os.path.join(self.tmp_dir, 'DL', 'TEST.foo'), version_id=invalid_version_id)
def test_upload(self): s3.cp(self.local_file, self.remote_file("unencrypted.txt"), encrypt=False) self.assertFalse( s3.info(self.remote_file("unencrypted.txt"))['encrypted']) s3.cp(self.local_file, self.remote_file("encrypted.txt")) # default now to encrypt self.assertTrue( s3.info(self.remote_file("encrypted.txt"))['encrypted'])
def test_s3_cp_download(self): s3.cp(self.existing_remote_file, os.path.join(self.tmp_dir, 'DL', 'TEST.foo')) self.assertTrue( os.path.exists(os.path.join(self.tmp_dir, 'DL', 'TEST.foo'))) s3.cp(self.existing_remote_file, os.path.join(self.tmp_dir, 'DL')) self.assertTrue( os.path.exists( os.path.join(self.tmp_dir, 'DL', s3.path.basename(self.existing_remote_file))))
def test_s3_with_double_slashes_in_key(self): ''' boto has a nasty behavior by default where it collapses `//` to `/` in keys ''' s3.cp(self.local_file, self.remote_file('double//slashes//bork//boto.foo')) self.assertEqual([self.remote_file('double//slashes//bork//boto.foo')], list( s3.ls(self.remote_file(''), return_full_urls=True)))
def test_s3_cp_gzip(self): s3.cp(self.local_file, self.remote_file("big.md")) s3.cp(self.local_file, self.remote_file("small.md"), gzip=True) self.assertNotEqual( s3.info(self.remote_file("big.md"))['content_encoding'], 'gzip') self.assertEqual( s3.info(self.remote_file("small.md"))['content_encoding'], 'gzip') self.assertLess( s3.info(self.remote_file("small.md"))['size'], s3.info(self.remote_file("big.md"))['size'])
def test_etag_on_multipart_upload(self): five_mb = 5 * 1024 * 1024 big_local_file = create_random_temporary_file(int(five_mb + 1024)) self.assertGreater(os.path.getsize(big_local_file), five_mb) remote_multipart = self.remote_file("TestEtag/multipart.md") s3.cp(big_local_file, remote_multipart, max_size=five_mb) self.assertIn("-", s3.etag(remote_multipart)) self.assertNotIn("-", s3.etag(big_local_file)) self.assertTrue( s3.etag_matches(big_local_file, s3.etag(remote_multipart))) os.remove(big_local_file)
def test_s3_ls(self): files = ["foo", "bar.baz", "quack/foo.foo"] for f in files: s3.cp(self.local_file, self.remote_file(f)) self.assertEqual(set(s3.ls(self.s3_test_location)), set(map(lambda x: self.s3_path + x, files))) self.assertEqual( set(s3.ls(self.s3_test_location, return_full_urls=True)), set([self.remote_file(x) for x in files])) self.assertEqual( set(s3.ls(self.s3_test_location, shallow=True)), set([self.s3_path + x for x in ['foo', 'bar.baz', 'quack/']]))
def test_downloads_from_s3_are_atomic_under_exceptions( self, download_mock): download_mock.side_effect = ValueError() # Now when the call to download the file is made, an exception will be thrown. # ideally, we'd throw it "in" boto via a mock, but we really want to test that # the file doesn't get written, so let's go ahead and let boto do the download # and then throw the exception in the validation with self.assertRaises(ValueError): s3.cp(self.existing_remote_file, os.path.join(self.tmp_dir, 'erroneous.foo'), validate=True) self.assertFalse( os.path.exists(os.path.join(self.tmp_dir, 'erroneous.foo')))
def test_s3_rm(self): for path in [ os.path.join(self.tmp_dir, 'foo'), self.remote_file("foo") ]: s3.cp(self.local_file, path) self.assert_s3_exists(path) s3.rm(path) self.assert_s3_does_not_exist(path) with self.assertRaises(s3.KeyNotFound): s3.rm(path) self.assertRaises(s3.InvalidSchemeException, s3.rm, ("http://example.com/foo"))
def existing_versioned_remote_file(self): # use a hardcoded path for test versioned file on S3 # to avoid bookkeeping # the current test won't make versioned copies of the file # the remote object will be either deleted (which will be overwritten later) # or download to local uri = 's3://{}/FOO/A_preexisting_file.md'.format(VERSIONED_TEST_BUCKET) if not s3.exists(uri): s3.cp(self.local_file, uri) return uri
def existing_versioned_remote_file(self): # use a hardcoded path for test versioned file on S3 # to avoid bookkeeping # the current test won't make versioned copies of the file # the remote object will be either deleted (which will be overwritten later) # or download to local uri = 's3://baiji-test-versioned/FOO/A_preexisting_file.md' if not s3.exists(uri): s3.cp(self.local_file, uri) return uri
def add(self, path, local_file, version=None, verbose=False): path = self.normalize_path(path) if self.is_versioned(path): raise ValueError('{} is already versioned; did you mean vc.update?'.format(path)) if version is None: version = '1.0.0' else: version = self.normalize_version_number(version) if not self.version_number_is_valid(version): raise ValueError('invalid version {}, always use versions of the form N.N.N'.format(version)) s3.cp(local_file, self.uri(path, version), progress=verbose) self.update_manifest(path, version)
def test_does_check_after_timeout(self): import time self.cache(self.filename) s3.cp(self.get_test_file_path(), self.remote_file, force=True) time.sleep(2) with mock.patch('baiji.s3.cp') as mock_cp: mock_cp.return_value = True self.cache(self.filename) mock_cp.assert_called_with( self.remote_file, self.local_file, progress=False, force=True, validate=True)
def setUp(self): import uuid from bltest.random_data import create_random_temporary_file super(TestAssetCache, self).setUp() self.filename = 'test_sc/{}/test_sample.txt'.format(uuid.uuid4()) self.local_file = os.path.join(self.cache_dir, self.bucket, self.filename) self.timestamp_file = os.path.join( self.cache_dir, '.timestamps', self.bucket, self.filename) self.remote_file = 's3://{}/{}'.format(self.bucket, self.filename) self.temp_file = create_random_temporary_file() s3.cp(self.temp_file, self.remote_file)
def test_s3_open_write_remote_file_with_context_manager(self): remote_file_name = self.remote_file("write_test_1") local_file_name = os.path.join(self.tmp_dir, "write_test_1") self.assert_s3_does_not_exist(remote_file_name) with s3.open(remote_file_name, 'w') as f: tempname = f.name f.write(self.truth) self.assertFalse(os.path.exists(tempname)) self.assert_s3_exists(remote_file_name) # download and confirm that it contains the correct contents s3.cp(remote_file_name, local_file_name) with open(local_file_name) as f: self.assertEqual(self.truth, f.read())
def main(self, src, dst): kwargs = { 'force': self.force, 'progress': self.progress, 'policy': self.policy, 'preserve_acl': self.preserve_acl, 'encoding': self.encoding, 'encrypt': self.encrypt, 'gzip': self.gzip, 'skip': self.skip, 'version_id': self.version_id, } if self.recursive or self.recursive_parallel: s3.cp_r(src, dst, parallel=self.recursive_parallel, **kwargs) else: s3.cp(src, dst, **kwargs)
def test_s3_cp_preserve_acl(self): s3.cp(self.local_file, self.remote_file("also_public.md"), policy='public-read') s3.cp(self.remote_file("also_public.md"), self.remote_file("still_public.md"), preserve_acl=True) self.assert_is_public(self.remote_file("also_public.md"), is_public=True) s3.cp(self.remote_file("also_public.md"), self.remote_file("no_longer_public.md")) self.assert_is_public(self.remote_file("no_longer_public.md"), is_public=False) with self.assertRaises(ValueError): s3.cp(self.remote_file("also_public.md"), os.path.join(self.tmp_dir, 'NoCanDo.txt'), preserve_acl=True)
def test_s3_cp_content_encoding(self): s3.cp(self.local_file, self.remote_file("encoded.md"), encoding='gzip') self.assertEqual(s3.info(self.remote_file("encoded.md"))['content_encoding'], 'gzip') s3.cp(self.local_file, self.remote_file("notencoded.md")) # just make sure gzip isn't the default ;) self.assertNotEqual(s3.info(self.remote_file("notencoded.md"))['content_encoding'], 'gzip') s3.cp(self.remote_file("encoded.md"), self.remote_file("still_encoded.md")) self.assertEqual(s3.info(self.remote_file("still_encoded.md"))['content_encoding'], 'gzip') s3.cp(self.remote_file("notencoded.md"), self.remote_file("now_encoded.md"), encoding='gzip') self.assertEqual(s3.info(self.remote_file("now_encoded.md"))['content_encoding'], 'gzip')