def main(): args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS) config = args.config external_source = False if 'external_id' in config: args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS_EXTERNAL_SOURCE) config = args.config external_source = True config['tables'] = validate_table_config(config) # If external_id is provided, we are trying to access files in another AWS account, and need to assume the role if external_source: s3.setup_aws_client(config) # Otherwise, confirm that we can access the bucket in our own AWS account else: try: for page in s3.list_files_in_bucket(config['bucket']): break except BaseException as err: LOGGER.error(err) if args.discover: do_discover(args.config) elif args.properties: do_sync(config, args.properties, args.state)
def main(): args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS) config = args.config config['tables'] = validate_table_config(config) s3.setup_aws_client(config) if args.discover: do_discover(args.config) elif args.properties: do_sync(config, args.properties, args.state)
def test_profile_based_auth(self): """Test AWS profile based authentication rather than access keys""" try: # Save original config to restore later orig_config = self.config.copy() # Remove access keys from config and add profile name del self.config['aws_access_key_id'] del self.config['aws_secret_access_key'] self.config['aws_profile'] = 'fake-profile' # Create a new S3 client using env vars with self.assertRaises(botocore.exceptions.ProfileNotFound): s3.setup_aws_client(self.config) # Restore the original state to not confuse other tests finally: self.config = orig_config.copy()
def main(): args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS) config = args.config config['tables'] = validate_table_config(config) try: for page in s3.list_files_in_bucket(config['bucket']): break LOGGER.warning( "I have direct access to the bucket without assuming the configured role." ) except: s3.setup_aws_client(config) if args.discover: do_discover(args.config) elif args.properties: do_sync(config, args.properties, args.state)
def test_profile_based_auth_aws_env_vars(self): """Test AWS profile based authentication using AWS environment variables""" try: # Save original config to restore later orig_config = self.config.copy() # Remove access keys from config and add profile name del self.config['aws_access_key_id'] del self.config['aws_secret_access_key'] # Profile name defined as env var and config is empty os.environ['AWS_PROFILE'] = 'fake_profile' # Create a new S3 client using env vars with self.assertRaises(botocore.exceptions.ProfileNotFound): s3.setup_aws_client(self.config) # Restore the original state to not confuse other tests finally: del os.environ['AWS_PROFILE'] self.config = orig_config.copy()
def main() -> None: """ Main function :return: None """ # We observed data who's field size exceeded the default maximum of # 131072. We believe the primary consequence of the following setting # is that a malformed, wide CSV would potentially parse into a single # large field rather than giving this error, but we also think the # chances of that are very small and at any rate the source data would # need to be fixed. The other consequence of this could be larger # memory consumption but that's acceptable as well. csv.field_size_limit(sys.maxsize) # Mock out `csv.field_size_limit` since messytables sets it... # TODO: replace messytables _field_size_limit = csv.field_size_limit csv.field_size_limit = lambda size: None args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS) config = args.config # Reassign the config tables to the validated object config['tables'] = CONFIG_CONTRACT(config.get('tables', {})) try: for _ in s3.list_files_in_bucket(config['bucket']): break LOGGER.warning( "I have direct access to the bucket without assuming the configured role." ) except Exception: s3.setup_aws_client(config) if args.discover: do_discover(args.config) elif args.properties: do_sync(config, args.properties, args.state) csv.field_size_limit = _field_size_limit
def test_credentials_aws_env_vars(self): """Test connecting to S3 with credentials defined in AWS environment variables rather than explicitly provided access keys""" try: # Save original config to restore later orig_config = self.config.copy() # Move aws access key and secret from config into environment variables os.environ['AWS_ACCESS_KEY_ID'] = self.config['aws_access_key_id'] os.environ['AWS_SECRET_ACCESS_KEY'] = self.config['aws_secret_access_key'] del self.config['aws_access_key_id'] del self.config['aws_secret_access_key'] # Create a new S3 client using env vars s3.setup_aws_client(self.config) self.assertListFiles() # Restore the original state to not confuse other tests finally: del os.environ['AWS_ACCESS_KEY_ID'] del os.environ['AWS_SECRET_ACCESS_KEY'] self.config = orig_config.copy()
def test_credentials(self): """Test connecting to S3 with credentials explicitly defined in config""" s3.setup_aws_client(self.config) self.assertListFiles()