def _download_file_or_folder_from_s3(self, resources_info: Dict) -> None:
s3_service = S3(resources_info)
s3_file_list = s3_service.get_bucket_file_list()
for s3_file in s3_file_list:
# Avoid download s3 'folders'
if not s3_file.endswith('/'):
file_path = self._get_download_file_path(file_key=s3_file)
# make sure the path folders are created
dir_path = os.path.dirname(file_path)
if dir_path and not os.path.isdir(dir_path):
os.makedirs(dir_path, exist_ok=True)
bucket, _ = get_bucket_and_folders(resources_info.get('lambda').get('input')[0].get('path'))
s3_service.download_file(bucket, s3_file, file_path)
def _process_s3_input_bucket_calls(self, resources_info: Dict, storage: Dict) -> None:
s3_service = S3(resources_info)
lambda_service = Lambda(resources_info)
s3_file_list = s3_service.get_bucket_file_list(storage)
bucket_name, _ = get_bucket_and_folders(storage.get('path'))
logger.info(f"Files found: '{s3_file_list}'")
# First do a request response invocation to prepare the lambda environment
if s3_file_list:
s3_event = s3_service.get_s3_event(bucket_name, s3_file_list.pop(0))
lambda_service.launch_request_response_event(s3_event)
# If the list has more elements, invoke functions asynchronously
if s3_file_list:
s3_event_list = s3_service.get_s3_event_list(bucket_name, s3_file_list)
lambda_service.process_asynchronous_lambda_invocations(s3_event_list)