def setUp(self):
self.local_file = os.path.abspath(".") + os.sep + "some_directory" + os.sep + "text1.txt"
self.local_dir = os.path.abspath(".") + os.sep + "some_directory" + os.sep
self.session = FakeSession()
self.service = self.session.get_service("s3")
self.endpoint = self.service.get_endpoint("us-east-1")
self.files = make_loc_files()
def setUp(self):
self.session = botocore.session.get_session(EnvironmentVariables)
self.loc_files = make_loc_files()
self.bucket = make_s3_files(self.session)
parser = argparse.ArgumentParser()
parser.add_argument('--region')
self.parsed_globals = parser.parse_args(['--region', 'us-east-1'])
def setUp(self):
self.local_file = os.path.abspath('.') + os.sep + 'some_directory' \
+ os.sep + 'text1.txt'
self.local_dir = os.path.abspath('.') + os.sep + 'some_directory' \
+ os.sep
self.session = FakeSession()
self.files = make_loc_files()
def setUp(self):
super(S3HandlerTestDeleteList, self).setUp()
self.session = FakeSession()
params = {'region': 'us-east-1'}
self.s3_handler = S3Handler(self.session, params)
self.bucket = make_s3_files(self.session)
self.loc_files = make_loc_files()
def setUp(self):
self.session = botocore.session.get_session(EnvironmentVariables)
self.service = self.session.get_service("s3")
self.endpoint = self.service.get_endpoint("us-east-1")
params = {"region": "us-east-1"}
self.s3_handler = S3Handler(self.session, params)
self.bucket = make_s3_files(self.session)
self.loc_files = make_loc_files()
def setUp(self):
self.environ = {}
self.environ_patch = patch('os.environ', self.environ)
self.environ_patch.start()
self.mock = MagicMock()
self.mock.get_config = MagicMock(return_value={'region': None})
self.loc_files = make_loc_files()
self.bucket = 's3testbucket'
def setUp(self):
super(S3HandlerTestDelete, self).setUp()
params = {'region': 'us-east-1'}
self.s3_handler = S3Handler(self.session, params,
runtime_config=runtime_config(
max_concurrent_requests=1))
self.loc_files = make_loc_files(self.file_creator)
self.bucket = 'mybucket'
def setUp(self):
self.session = botocore.session.get_session(EnvironmentVariables)
self.service = self.session.get_service('s3')
self.endpoint = self.service.get_endpoint('us-east-1')
params = {'region': 'us-east-1'}
self.s3_handler = S3Handler(self.session, params)
self.bucket = make_s3_files(self.session)
self.loc_files = make_loc_files()
def setUp(self):
super(CommandArchitectureTest, self).setUp()
self.session = FakeSession()
self.bucket = make_s3_files(self.session)
self.loc_files = make_loc_files()
self.output = StringIO()
self.saved_stdout = sys.stdout
sys.stdout = self.output
def setUp(self):
self.environ = {}
self.environ_patch = patch("os.environ", self.environ)
self.environ_patch.start()
self.session = FakeSession()
self.mock = MagicMock()
self.mock.get_config = MagicMock(return_value={"region": None})
self.loc_files = make_loc_files()
self.bucket = make_s3_files(self.session)
def setUp(self):
super(S3HandlerTestDeleteList, self).setUp()
self.session = FakeSession()
self.service = self.session.get_service('s3')
self.endpoint = self.service.get_endpoint('us-east-1')
params = {'region': 'us-east-1'}
self.s3_handler = S3Handler(self.session, params)
self.bucket = make_s3_files(self.session)
self.loc_files = make_loc_files()
def setUp(self):
super(S3HandlerTestMvLocalS3, self).setUp()
self.session = FakeSession()
params = {'region': 'us-east-1', 'acl': ['private']}
self.s3_handler = S3Handler(self.session, params)
self.bucket = create_bucket(self.session)
self.loc_files = make_loc_files()
self.s3_files = [self.bucket + '/text1.txt',
self.bucket + '/another_directory/text2.txt']
def setUp(self):
self.local_file = six.text_type(os.path.abspath('.') + os.sep + 'some_directory' \
+ os.sep + 'text1.txt')
self.local_dir = six.text_type(os.path.abspath('.') + os.sep + 'some_directory' \
+ os.sep)
self.session = FakeSession()
self.service = self.session.get_service('s3')
self.endpoint = self.service.get_endpoint('us-east-1')
self.files = make_loc_files()
def setUp(self):
super(S3HandlerExceptionSingleTaskTest, self).setUp()
self.session = FakeSession(True, True)
params = {'region': 'us-east-1'}
self.s3_handler = S3Handler(self.session, params)
self.bucket = create_bucket(self.session)
self.loc_files = make_loc_files()
self.s3_files = [self.bucket + '/text1.txt',
self.bucket + '/another_directory/text2.txt']
def setUp(self):
self.session = botocore.session.get_session(EnvironmentVariables)
self.client = self.session.create_client('s3', 'us-west-2')
self.source_client = self.session.create_client('s3', 'us-west-2')
params = {'region': 'us-west-2'}
self.s3_handler = S3Handler(self.session, params)
self.bucket = make_s3_files(self.session)
self.file_creator = FileCreator()
self.loc_files = make_loc_files(self.file_creator)
def setUp(self):
super(S3HandlerTestMvLocalS3, self).setUp()
params = {'region': 'us-east-1', 'acl': ['private'], 'quiet': True}
self.s3_handler = S3Handler(self.session, params,
runtime_config=runtime_config(
max_concurrent_requests=1))
self.bucket = 'mybucket'
self.loc_files = make_loc_files(self.file_creator)
self.s3_files = [self.bucket + '/text1.txt',
self.bucket + '/another_directory/text2.txt']
def setUp(self):
super(S3HandlerExceptionMultiTaskTest, self).setUp()
self.session = FakeSession(True, True)
params = {'region': 'us-east-1'}
self.s3_handler_multi = S3Handler(self.session, params,
multi_threshold=10, chunksize=2)
self.bucket = create_bucket(self.session)
self.loc_files = make_loc_files()
self.s3_files = [self.bucket + '/text1.txt',
self.bucket + '/another_directory/text2.txt']
def setUp(self):
super(S3HandlerTestMvLocalS3, self).setUp()
self.session = FakeSession()
self.service = self.session.get_service('s3')
self.endpoint = self.service.get_endpoint('us-east-1')
params = {'region': 'us-east-1', 'acl': ['private'], 'quiet': True}
self.s3_handler = S3Handler(self.session, params)
self.bucket = create_bucket(self.session)
self.loc_files = make_loc_files()
self.s3_files = [self.bucket + '/text1.txt',
self.bucket + '/another_directory/text2.txt']
def setUp(self):
super(CommandArchitectureTest, self).setUp()
self.session = self.driver.session
self.bucket = 'mybucket'
self.file_creator = FileCreator()
self.loc_files = make_loc_files(self.file_creator)
self.output = StringIO()
self.err_output = StringIO()
self.saved_stdout = sys.stdout
self.saved_stderr = sys.stderr
sys.stdout = self.output
sys.stderr = self.err_output
def setUp(self):
super(S3HandlerTestUpload, self).setUp()
self.session = FakeSession()
params = {'region': 'us-east-1', 'acl': ['private']}
self.s3_handler = S3Handler(self.session, params)
self.s3_handler_multi = S3Handler(self.session, multi_threshold=10,
chunksize=2,
params=params)
self.bucket = create_bucket(self.session)
self.loc_files = make_loc_files()
self.s3_files = [self.bucket + '/text1.txt',
self.bucket + '/another_directory/text2.txt']
def setUp(self):
self.session = botocore.session.get_session(EnvironmentVariables)
self.service = self.session.get_service("s3")
self.endpoint = self.service.get_endpoint("us-east-1")
params = {"region": "us-east-1", "acl": ["private"]}
self.s3_handler = S3Handler(self.session, params)
self.s3_handler_multi = S3Handler(self.session, multi_threshold=10, chunksize=2, params=params)
self.bucket = create_bucket(self.session)
self.loc_files = make_loc_files()
self.s3_files = [self.bucket + "/text1.txt", self.bucket + "/another_directory/text2.txt"]
self.output = StringIO()
self.saved_stdout = sys.stdout
sys.stdout = self.output
def setUp(self):
super(S3HandlerExceptionMultiTaskTest, self).setUp()
self.session = FakeSession(True, True)
self.service = self.session.get_service('s3')
self.endpoint = self.service.get_endpoint('us-east-1')
params = {'region': 'us-east-1', 'quiet': True}
self.s3_handler_multi = S3Handler(
self.session, params,
runtime_config=runtime_config(
multipart_threshold=10, multipart_chunksize=2))
self.bucket = create_bucket(self.session)
self.loc_files = make_loc_files()
self.s3_files = [self.bucket + '/text1.txt',
self.bucket + '/another_directory/text2.txt']
def setUp(self):
self.session = botocore.session.get_session(EnvironmentVariables)
params = {'region': 'us-east-1', 'acl': ['private']}
self.s3_handler = S3Handler(self.session, params)
self.s3_handler_multi = S3Handler(self.session, multi_threshold=10,
chunksize=2,
params=params)
self.bucket = create_bucket(self.session)
self.loc_files = make_loc_files()
self.s3_files = [self.bucket + '/text1.txt',
self.bucket + '/another_directory/text2.txt']
self.output = StringIO()
self.saved_stdout = sys.stdout
sys.stdout = self.output
def setUp(self):
super(S3HandlerTestUpload, self).setUp()
params = {'region': 'us-east-1', 'acl': 'private', 'quiet': True}
self.s3_handler = S3Handler(
self.session, params, runtime_config=runtime_config(
max_concurrent_requests=1))
self.s3_handler_multi = S3Handler(
self.session, params=params,
runtime_config=runtime_config(
multipart_threshold=10, multipart_chunksize=10,
max_concurrent_requests=1))
self.bucket = 'mybucket'
self.loc_files = make_loc_files(self.file_creator)
self.s3_files = [self.bucket + '/text1.txt',
self.bucket + '/another_directory/text2.txt']
def setUp(self):
super(S3HandlerExceptionMultiTaskTest, self).setUp()
self.session = FakeSession(True, True)
self.service = self.session.get_service('s3')
self.endpoint = self.service.get_endpoint('us-east-1')
params = {'region': 'us-east-1'}
self.s3_handler_multi = S3Handler(self.session,
params,
multi_threshold=10,
chunksize=2)
self.bucket = create_bucket(self.session)
self.loc_files = make_loc_files()
self.s3_files = [
self.bucket + '/text1.txt',
self.bucket + '/another_directory/text2.txt'
]
def setUp(self):
self.session = botocore.session.get_session(EnvironmentVariables)
self.service = self.session.get_service('s3')
self.endpoint = self.service.get_endpoint('us-east-1')
params = {'region': 'us-east-1', 'acl': ['private']}
self.s3_handler = S3Handler(self.session, params)
self.s3_handler_multi = S3Handler(self.session, multi_threshold=10,
chunksize=2,
params=params)
self.bucket = create_bucket(self.session)
self.loc_files = make_loc_files()
self.s3_files = [self.bucket + '/text1.txt',
self.bucket + '/another_directory/text2.txt']
self.output = StringIO()
self.saved_stdout = sys.stdout
sys.stdout = self.output
def setUp(self):
self.session = botocore.session.get_session(EnvironmentVariables)
self.client = self.session.create_client('s3', 'us-west-2')
params = {'region': 'us-west-2', 'acl': ['private']}
self.s3_handler = S3Handler(self.session, params)
self.s3_handler_multi = S3Handler(
self.session, params=params,
runtime_config=runtime_config(
multipart_threshold=10, multipart_chunksize=2))
self.bucket = create_bucket(self.session)
self.file_creator = FileCreator()
self.loc_files = make_loc_files(self.file_creator)
self.s3_files = [self.bucket + '/text1.txt',
self.bucket + '/another_directory/text2.txt']
self.output = StringIO()
self.saved_stderr = sys.stderr
sys.stderr = self.output
def setUp(self):
super(S3HandlerTestUpload, self).setUp()
self.session = FakeSession()
self.service = self.session.get_service('s3')
self.endpoint = self.service.get_endpoint('us-east-1')
params = {'region': 'us-east-1', 'acl': ['private']}
self.s3_handler = S3Handler(self.session, params)
self.s3_handler_multi = S3Handler(self.session,
multi_threshold=10,
chunksize=2,
params=params)
self.bucket = create_bucket(self.session)
self.loc_files = make_loc_files()
self.s3_files = [
self.bucket + '/text1.txt',
self.bucket + '/another_directory/text2.txt'
]
def setUp(self):
self.session = botocore.session.get_session(EnvironmentVariables)
self.client = self.session.create_client('s3', 'us-west-2')
params = {'region': 'us-west-2', 'acl': ['private']}
self.s3_handler = S3Handler(self.session, params)
self.s3_handler_multi = S3Handler(
self.session, params=params,
runtime_config=runtime_config(
multipart_threshold=10, multipart_chunksize=2))
self.bucket = create_bucket(self.session)
self.file_creator = FileCreator()
self.loc_files = make_loc_files(self.file_creator)
self.s3_files = [self.bucket + '/text1.txt',
self.bucket + '/another_directory/text2.txt']
self.output = StringIO()
self.saved_stderr = sys.stderr
sys.stderr = self.output
def setUp(self):
super(S3HandlerTestUpload, self).setUp()
params = {'region': 'us-east-1', 'acl': 'private', 'quiet': True}
self.s3_handler = S3Handler(
self.session,
params,
runtime_config=runtime_config(max_concurrent_requests=1))
self.s3_handler_multi = S3Handler(self.session,
params=params,
runtime_config=runtime_config(
multipart_threshold=10,
multipart_chunksize=10,
max_concurrent_requests=1))
self.bucket = 'mybucket'
self.loc_files = make_loc_files(self.file_creator)
self.s3_files = [
self.bucket + '/text1.txt',
self.bucket + '/another_directory/text2.txt'
]
def test_upload(self):
self.loc_files = make_loc_files(self.file_creator)
# Confirm there are no objects in the bucket.
response = self.client.list_objects(Bucket=self.bucket)
self.assertEqual(len(response.get('Contents', [])), 0)
# Create file info objects to perform upload.
files = [self.loc_files[0], self.loc_files[1]]
tasks = []
for i in range(len(files)):
tasks.append(FileInfo(
src=self.loc_files[i],
dest=self.s3_files[i],
operation_name='upload', size=0,
client=self.client,
))
# Perform the upload.
self.s3_handler.call(tasks)
# Confirm the files were uploaded.
response = self.client.list_objects(Bucket=self.bucket)
self.assertEqual(len(response.get('Contents', [])), 2)
def test_upload(self):
self.loc_files = make_loc_files(self.file_creator)
# Confirm there are no objects in the bucket.
response = self.client.list_objects(Bucket=self.bucket)
self.assertEqual(len(response.get('Contents', [])), 0)
# Create file info objects to perform upload.
files = [self.loc_files[0], self.loc_files[1]]
tasks = []
for i in range(len(files)):
tasks.append(FileInfo(
src=self.loc_files[i],
dest=self.s3_files[i],
operation_name='upload', size=0,
client=self.client,
))
# Perform the upload.
self.s3_handler.call(tasks)
# Confirm the files were uploaded.
response = self.client.list_objects(Bucket=self.bucket)
self.assertEqual(len(response.get('Contents', [])), 2)
def test_multi_upload(self):
self.loc_files = make_loc_files(self.file_creator, self.threshold+1)
files = [self.loc_files[0], self.loc_files[1]]
tasks = []
for i in range(len(files)):
tasks.append(FileInfo(
src=self.loc_files[i],
dest=self.s3_files[i],
size=self.threshold+1,
operation_name='upload',
client=self.client,
))
self.s3_handler_multi.call(tasks)
# Confirm UploadPart was called
self.assertIn("Completed 4 of 4 part(s)", self.output.getvalue())
# Confirm the files were uploaded.
response = self.client.list_objects(Bucket=self.bucket)
self.assertEqual(len(response.get('Contents', [])), 2)
def test_multi_upload(self):
self.loc_files = make_loc_files(self.file_creator, self.threshold+1)
files = [self.loc_files[0], self.loc_files[1]]
tasks = []
for i in range(len(files)):
tasks.append(FileInfo(
src=self.loc_files[i],
dest=self.s3_files[i],
size=self.threshold+1,
operation_name='upload',
client=self.client,
))
self.s3_handler_multi.call(tasks)
# Confirm UploadPart was called
self.assertIn("Completed 4 of 4 part(s)", self.output.getvalue())
# Confirm the files were uploaded.
response = self.client.list_objects(Bucket=self.bucket)
self.assertEqual(len(response.get('Contents', [])), 2)
def setUp(self):
self.session = botocore.session.get_session(EnvironmentVariables)
params = {'region': 'us-east-1'}
self.s3_handler = S3Handler(self.session, params)
self.bucket = make_s3_files(self.session)
self.loc_files = make_loc_files()
def setUp(self):
self.session = botocore.session.get_session(EnvironmentVariables)
params = {'region': 'us-east-1'}
self.s3_handler = S3Handler(self.session, params)
self.bucket = make_s3_files(self.session, key1='a+b/foo', key2=None)
self.loc_files = make_loc_files()
def setUp(self):
self.client = None
self.file_creator = FileCreator()
self.files = make_loc_files(self.file_creator)
self.local_file = self.files[0]
self.local_dir = self.files[3] + os.sep
def setUp(self):
self.client = None
self.file_creator = FileCreator()
self.files = make_loc_files(self.file_creator)
self.local_file = self.files[0]
self.local_dir = self.files[3] + os.sep