def download_dir(self): logging.debug('Downloading s3 object: "%s" Target: %s', self.bucket + "/" + self.file_path, self.path) client = boto3.client('s3', endpoint_url=extract_endpoint()) if not self.file_path.endswith('/'): self.file_path += '/' objects = client.list_objects_v2(Bucket=self.bucket, Prefix=self.file_path) # If the file path does not exists in s3 bucket, 'Contents' key will not be present in objects if "Contents" not in objects: logging.error('Got status code: %s', 404) logging.error("Invalid file path!.") return 1 # Looping through the list of objects and downloading them for obj in objects["Contents"]: file_name = os.path.basename(obj["Key"]) dir_name = os.path.dirname(obj["Key"]) path_to_create = re.sub( r'^' + self.file_path.strip('/').replace('/', '\/') + '', "", dir_name).strip('/') path_to_create = os.path.join(self.path, path_to_create) os.makedirs(path_to_create, exist_ok=True) if self.get_s3_file(os.path.join(path_to_create, file_name), obj["Key"]): return 1 return 0
def test_unset_env_var(self): """ In case the 'AWS_CONFIG_FILE' env var is not set and there's no config file in the default location, the result should be 'None'. """ exists = unittest.mock exists.patch("os.path.exists", return_value=False) os.environ.pop("AWS_CONFIG_FILE") self.assertEqual(extract_endpoint(), None)
def test_extract_url_from_config_file(mock_path_exists): """ Testing extraction of endpoint url from default file location """ read_data = '\n'.join( ["[default]", "endpoint_url = http://s3-aws-region.amazonaws.com"]) with patch("builtins.open", mock_open(read_data=read_data), create=True) as mock_file: mock_file.return_value.__iter__.return_value = read_data.splitlines() assert extract_endpoint() == "http://s3-aws-region.amazonaws.com" mock_file.assert_called_once_with("~/.aws/config", encoding=None)
def test_extract_url_from_environ_variable(): """ Testing successful extraction of endpoint url read from file path saved on enviornment variable """ read_data = '\n'.join( ["[default]", "endpoint_url = http://s3-aws-region.amazonaws.com"]) with patch("builtins.open", mock_open(read_data=read_data), create=True) as mock_file: mock_file.return_value.__iter__.return_value = read_data.splitlines() assert (extract_endpoint() == "http://s3-aws-region.amazonaws.com") mock_file.assert_called_once_with(os.environ["AWS_CONFIG_FILE"], encoding=None)
def test_default_profile(self): self.assertEqual(extract_endpoint(), "http://foo.bar")
def test_non_existent_profile(self): """ In case a profile does not exist, 'None' should be returned. """ self.assertEqual(extract_endpoint("random_profile"), None)
def test_other_profile(self): self.assertEqual(extract_endpoint("other_profile"), "http://other.endpoint")
def __enter__(self): client = boto3.resource('s3', endpoint_url=extract_endpoint()) if self.check_if_bucket_exists(client): sys.exit(1) self.bucket_obj = client.Bucket(self.bucket) return self