def download_dir(self):
logging.debug('Downloading s3 object: "%s" Target: %s',
self.bucket + "/" + self.file_path, self.path)
client = boto3.client('s3', endpoint_url=extract_endpoint())
if not self.file_path.endswith('/'):
self.file_path += '/'
objects = client.list_objects_v2(Bucket=self.bucket,
Prefix=self.file_path)
# If the file path does not exists in s3 bucket, 'Contents' key will not be present in objects
if "Contents" not in objects:
logging.error('Got status code: %s', 404)
logging.error("Invalid file path!.")
return 1
# Looping through the list of objects and downloading them
for obj in objects["Contents"]:
file_name = os.path.basename(obj["Key"])
dir_name = os.path.dirname(obj["Key"])
path_to_create = re.sub(
r'^' + self.file_path.strip('/').replace('/', '\/') + '', "",
dir_name).strip('/')
path_to_create = os.path.join(self.path, path_to_create)
os.makedirs(path_to_create, exist_ok=True)
if self.get_s3_file(os.path.join(path_to_create, file_name),
obj["Key"]):
return 1
return 0
def test_unset_env_var(self):
""" In case the 'AWS_CONFIG_FILE' env var is not set and there's no
config file in the default location, the result should be 'None'.
"""
exists = unittest.mock
exists.patch("os.path.exists", return_value=False)
os.environ.pop("AWS_CONFIG_FILE")
self.assertEqual(extract_endpoint(), None)
def test_extract_url_from_config_file(mock_path_exists):
"""
Testing extraction of endpoint url from default file location
"""
read_data = '\n'.join(
["[default]", "endpoint_url = http://s3-aws-region.amazonaws.com"])
with patch("builtins.open", mock_open(read_data=read_data),
create=True) as mock_file:
mock_file.return_value.__iter__.return_value = read_data.splitlines()
assert extract_endpoint() == "http://s3-aws-region.amazonaws.com"
mock_file.assert_called_once_with("~/.aws/config", encoding=None)
def test_extract_url_from_environ_variable():
"""
Testing successful extraction of endpoint url read from file path saved on enviornment variable
"""
read_data = '\n'.join(
["[default]", "endpoint_url = http://s3-aws-region.amazonaws.com"])
with patch("builtins.open", mock_open(read_data=read_data),
create=True) as mock_file:
mock_file.return_value.__iter__.return_value = read_data.splitlines()
assert (extract_endpoint() == "http://s3-aws-region.amazonaws.com")
mock_file.assert_called_once_with(os.environ["AWS_CONFIG_FILE"],
encoding=None)
def test_default_profile(self):
self.assertEqual(extract_endpoint(), "http://foo.bar")
def test_non_existent_profile(self):
""" In case a profile does not exist, 'None' should be returned. """
self.assertEqual(extract_endpoint("random_profile"), None)
def test_other_profile(self):
self.assertEqual(extract_endpoint("other_profile"),
"http://other.endpoint")
def __enter__(self):
client = boto3.resource('s3', endpoint_url=extract_endpoint())
if self.check_if_bucket_exists(client):
sys.exit(1)
self.bucket_obj = client.Bucket(self.bucket)
return self