def main(config, input_stream=None): with connect(user=config.get('snowflake_username'), password=config.get('snowflake_password'), role=config.get('snowflake_role'), authenticator=config.get('snowflake_authenticator', 'snowflake'), account=config.get('snowflake_account'), warehouse=config.get('snowflake_warehouse'), database=config.get('snowflake_database'), schema=config.get('snowflake_schema', 'PUBLIC'), autocommit=False) as connection: s3_config = config.get('target_s3') s3 = None if s3_config: s3 = S3(s3_config.get('aws_access_key_id'), s3_config.get('aws_secret_access_key'), s3_config.get('bucket'), s3_config.get('key_prefix')) target = SnowflakeTarget( connection, s3=s3, logging_level=config.get('logging_level'), persist_empty_tables=config.get('persist_empty_tables')) if input_stream: target_tools.stream_to_target(input_stream, target, config=config) else: target_tools.main(target)
def main(config, input_stream=None): with psycopg2.connect( connection_factory=MillisLoggingConnection, host=config.get('postgres_host', 'localhost'), port=config.get('postgres_port', 5432), dbname=config.get('postgres_database'), user=config.get('postgres_username'), password=config.get('postgres_password'), sslmode=config.get('postgres_sslmode'), sslcert=config.get('postgres_sslcert'), sslkey=config.get('postgres_sslkey'), sslrootcert=config.get('postgres_sslrootcert'), sslcrl=config.get('postgres_sslcrl'), application_name=config.get('application_name', 'target-postgres'), ) as connection: postgres_target = PostgresTarget( connection, postgres_schema=config.get('postgres_schema', 'public'), logging_level=config.get('logging_level'), persist_empty_tables=config.get('persist_empty_tables'), add_upsert_indexes=config.get('add_upsert_indexes', True), before_run_sql=config.get('before_run_sql'), after_run_sql=config.get('after_run_sql'), ) if input_stream: target_tools.stream_to_target(input_stream, postgres_target, config=config) else: target_tools.main(postgres_target)
def main(config, input_stream=None): with psycopg2.connect( connection_factory=MillisLoggingConnection, host=config.get('redshift_host'), port=config.get('redshift_port', 5439), dbname=config.get('redshift_database'), user=config.get('redshift_username'), password=config.get('redshift_password') ) as connection: s3_config = config.get('target_s3') s3 = S3(s3_config.get('aws_access_key_id'), s3_config.get('aws_secret_access_key'), s3_config.get('aws_session_token'), s3_config.get('region_name'), s3_config.get('bucket'), s3_config.get('key_prefix')) redshift_target = RedshiftTarget( connection, s3, redshift_schema=config.get('redshift_schema', 'public'), logging_level=config.get('logging_level'), default_column_length=config.get('default_column_length', 1000), persist_empty_tables=config.get('persist_empty_tables') ) if input_stream: target_tools.stream_to_target( input_stream, redshift_target, config=config) else: target_tools.main(redshift_target)
def main(config, input_stream=None): tunnel = None try: LOGGER.info(config) if bool(config.get('use_ssh_tunnel')) == True: LOGGER.info( f"use_ssh_tunnel is set to true; connecting to {config['redshift_host']}:{config['redshift_port']} via {config['ssh_jump_server']}:{config['ssh_jump_server_port']}" ) tunnel = sshtunnel.open_tunnel( (config['ssh_jump_server'], int( config['ssh_jump_server_port'])), ssh_username=config['ssh_username'], ssh_pkey=config['ssh_private_key_path'], ssh_private_key_password=config['ssh_private_key_password'] if 'ssh_private_key_password' in config else None, remote_bind_address=(config['redshift_host'], int(config['redshift_port']))) tunnel.start() time.sleep(1) config[ 'redshift_host'] = '127.0.0.1' # rewrite the config to go through the tunnel config['redshift_port'] = tunnel.local_bind_port else: LOGGER.debug( f"use_ssh_tunnel is not set or is false; connecting directly to {config['redshift_host']}:{config['redshift_port']}" ) with psycopg2.connect( connection_factory=MillisLoggingConnection, host=config.get('redshift_host'), port=config.get('redshift_port', 5439), dbname=config.get('redshift_database'), user=config.get('redshift_username'), password=config.get('redshift_password')) as connection: s3_config = config.get('target_s3') s3 = S3(s3_config.get('aws_access_key_id'), s3_config.get('aws_secret_access_key'), s3_config.get('bucket'), s3_config.get('key_prefix'), aws_session_token=s3_config.get('aws_session_token')) redshift_target = RedshiftTarget( connection, s3, redshift_schema=config.get('redshift_schema', 'public'), logging_level=config.get('logging_level'), default_column_length=config.get('default_column_length', 1000), persist_empty_tables=config.get('persist_empty_tables')) if input_stream: target_tools.stream_to_target(input_stream, redshift_target, config=config) else: target_tools.main(redshift_target) finally: if tunnel is not None: tunnel.stop()
def main(config, input_stream=None): with psycopg2.connect( host=config.get('postgres_host', 'localhost'), port=config.get('postgres_port', 5432), dbname=config.get('postgres_database'), user=config.get('postgres_username'), password=config.get('postgres_password')) as connection: postgres_target = PostgresTarget(connection, postgres_schema=config.get( 'postgres_schema', 'public')) if input_stream: target_tools.stream_to_target(input_stream, postgres_target, config=config) else: target_tools.main(postgres_target)
def main(config, input_stream=None): with psycopg2.connect( connection_factory=MillisLoggingConnection, host=config.get('postgres_host', 'localhost'), port=config.get('postgres_port', 5432), dbname=config.get('postgres_database'), user=config.get('postgres_username'), password=config.get('postgres_password')) as connection: postgres_target = PostgresTarget( connection, postgres_schema=config.get('postgres_schema', 'public'), logging_level=config.get('logging_level'), persist_empty_tables=config.get('persist_empty_tables')) if input_stream: target_tools.stream_to_target(input_stream, postgres_target, config=config) else: target_tools.main(postgres_target)