def main():
args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS)
config = args.config
external_source = False
if 'external_id' in config:
args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS_EXTERNAL_SOURCE)
config = args.config
external_source = True
config['tables'] = validate_table_config(config)
# If external_id is provided, we are trying to access files in another AWS account, and need to assume the role
if external_source:
s3.setup_aws_client(config)
# Otherwise, confirm that we can access the bucket in our own AWS account
else:
try:
for page in s3.list_files_in_bucket(config['bucket']):
break
except BaseException as err:
LOGGER.error(err)
if args.discover:
do_discover(args.config)
elif args.properties:
do_sync(config, args.properties, args.state)
def main():
args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS)
config = args.config
config['tables'] = validate_table_config(config)
s3.setup_aws_client(config)
if args.discover:
do_discover(args.config)
elif args.properties:
do_sync(config, args.properties, args.state)
def test_profile_based_auth(self):
"""Test AWS profile based authentication rather than access keys"""
try:
# Save original config to restore later
orig_config = self.config.copy()
# Remove access keys from config and add profile name
del self.config['aws_access_key_id']
del self.config['aws_secret_access_key']
self.config['aws_profile'] = 'fake-profile'
# Create a new S3 client using env vars
with self.assertRaises(botocore.exceptions.ProfileNotFound):
s3.setup_aws_client(self.config)
# Restore the original state to not confuse other tests
finally:
self.config = orig_config.copy()
def main():
args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS)
config = args.config
config['tables'] = validate_table_config(config)
try:
for page in s3.list_files_in_bucket(config['bucket']):
break
LOGGER.warning(
"I have direct access to the bucket without assuming the configured role."
)
except:
s3.setup_aws_client(config)
if args.discover:
do_discover(args.config)
elif args.properties:
do_sync(config, args.properties, args.state)
def test_profile_based_auth_aws_env_vars(self):
"""Test AWS profile based authentication using AWS environment variables"""
try:
# Save original config to restore later
orig_config = self.config.copy()
# Remove access keys from config and add profile name
del self.config['aws_access_key_id']
del self.config['aws_secret_access_key']
# Profile name defined as env var and config is empty
os.environ['AWS_PROFILE'] = 'fake_profile'
# Create a new S3 client using env vars
with self.assertRaises(botocore.exceptions.ProfileNotFound):
s3.setup_aws_client(self.config)
# Restore the original state to not confuse other tests
finally:
del os.environ['AWS_PROFILE']
self.config = orig_config.copy()
def main() -> None:
"""
Main function
:return: None
"""
# We observed data who's field size exceeded the default maximum of
# 131072. We believe the primary consequence of the following setting
# is that a malformed, wide CSV would potentially parse into a single
# large field rather than giving this error, but we also think the
# chances of that are very small and at any rate the source data would
# need to be fixed. The other consequence of this could be larger
# memory consumption but that's acceptable as well.
csv.field_size_limit(sys.maxsize)
# Mock out `csv.field_size_limit` since messytables sets it...
# TODO: replace messytables
_field_size_limit = csv.field_size_limit
csv.field_size_limit = lambda size: None
args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS)
config = args.config
# Reassign the config tables to the validated object
config['tables'] = CONFIG_CONTRACT(config.get('tables', {}))
try:
for _ in s3.list_files_in_bucket(config['bucket']):
break
LOGGER.warning(
"I have direct access to the bucket without assuming the configured role."
)
except Exception:
s3.setup_aws_client(config)
if args.discover:
do_discover(args.config)
elif args.properties:
do_sync(config, args.properties, args.state)
csv.field_size_limit = _field_size_limit
def test_credentials_aws_env_vars(self):
"""Test connecting to S3 with credentials defined in AWS environment variables
rather than explicitly provided access keys"""
try:
# Save original config to restore later
orig_config = self.config.copy()
# Move aws access key and secret from config into environment variables
os.environ['AWS_ACCESS_KEY_ID'] = self.config['aws_access_key_id']
os.environ['AWS_SECRET_ACCESS_KEY'] = self.config['aws_secret_access_key']
del self.config['aws_access_key_id']
del self.config['aws_secret_access_key']
# Create a new S3 client using env vars
s3.setup_aws_client(self.config)
self.assertListFiles()
# Restore the original state to not confuse other tests
finally:
del os.environ['AWS_ACCESS_KEY_ID']
del os.environ['AWS_SECRET_ACCESS_KEY']
self.config = orig_config.copy()
def test_credentials(self):
"""Test connecting to S3 with credentials explicitly defined in config"""
s3.setup_aws_client(self.config)
self.assertListFiles()