def main(config, input_stream=None):
with connect(user=config.get('snowflake_username'),
password=config.get('snowflake_password'),
role=config.get('snowflake_role'),
authenticator=config.get('snowflake_authenticator',
'snowflake'),
account=config.get('snowflake_account'),
warehouse=config.get('snowflake_warehouse'),
database=config.get('snowflake_database'),
schema=config.get('snowflake_schema', 'PUBLIC'),
autocommit=False) as connection:
s3_config = config.get('target_s3')
s3 = None
if s3_config:
s3 = S3(s3_config.get('aws_access_key_id'),
s3_config.get('aws_secret_access_key'),
s3_config.get('bucket'), s3_config.get('key_prefix'))
target = SnowflakeTarget(
connection,
s3=s3,
logging_level=config.get('logging_level'),
persist_empty_tables=config.get('persist_empty_tables'))
if input_stream:
target_tools.stream_to_target(input_stream, target, config=config)
else:
target_tools.main(target)
def main(config, input_stream=None):
with psycopg2.connect(
connection_factory=MillisLoggingConnection,
host=config.get('postgres_host', 'localhost'),
port=config.get('postgres_port', 5432),
dbname=config.get('postgres_database'),
user=config.get('postgres_username'),
password=config.get('postgres_password'),
sslmode=config.get('postgres_sslmode'),
sslcert=config.get('postgres_sslcert'),
sslkey=config.get('postgres_sslkey'),
sslrootcert=config.get('postgres_sslrootcert'),
sslcrl=config.get('postgres_sslcrl'),
application_name=config.get('application_name', 'target-postgres'),
) as connection:
postgres_target = PostgresTarget(
connection,
postgres_schema=config.get('postgres_schema', 'public'),
logging_level=config.get('logging_level'),
persist_empty_tables=config.get('persist_empty_tables'),
add_upsert_indexes=config.get('add_upsert_indexes', True),
before_run_sql=config.get('before_run_sql'),
after_run_sql=config.get('after_run_sql'),
)
if input_stream:
target_tools.stream_to_target(input_stream,
postgres_target,
config=config)
else:
target_tools.main(postgres_target)
def main(config, input_stream=None):
with psycopg2.connect(
connection_factory=MillisLoggingConnection,
host=config.get('redshift_host'),
port=config.get('redshift_port', 5439),
dbname=config.get('redshift_database'),
user=config.get('redshift_username'),
password=config.get('redshift_password')
) as connection:
s3_config = config.get('target_s3')
s3 = S3(s3_config.get('aws_access_key_id'),
s3_config.get('aws_secret_access_key'),
s3_config.get('aws_session_token'),
s3_config.get('region_name'),
s3_config.get('bucket'),
s3_config.get('key_prefix'))
redshift_target = RedshiftTarget(
connection,
s3,
redshift_schema=config.get('redshift_schema', 'public'),
logging_level=config.get('logging_level'),
default_column_length=config.get('default_column_length', 1000),
persist_empty_tables=config.get('persist_empty_tables')
)
if input_stream:
target_tools.stream_to_target(
input_stream, redshift_target, config=config)
else:
target_tools.main(redshift_target)
def main(config, input_stream=None):
tunnel = None
try:
LOGGER.info(config)
if bool(config.get('use_ssh_tunnel')) == True:
LOGGER.info(
f"use_ssh_tunnel is set to true; connecting to {config['redshift_host']}:{config['redshift_port']} via {config['ssh_jump_server']}:{config['ssh_jump_server_port']}"
)
tunnel = sshtunnel.open_tunnel(
(config['ssh_jump_server'], int(
config['ssh_jump_server_port'])),
ssh_username=config['ssh_username'],
ssh_pkey=config['ssh_private_key_path'],
ssh_private_key_password=config['ssh_private_key_password']
if 'ssh_private_key_password' in config else None,
remote_bind_address=(config['redshift_host'],
int(config['redshift_port'])))
tunnel.start()
time.sleep(1)
config[
'redshift_host'] = '127.0.0.1' # rewrite the config to go through the tunnel
config['redshift_port'] = tunnel.local_bind_port
else:
LOGGER.debug(
f"use_ssh_tunnel is not set or is false; connecting directly to {config['redshift_host']}:{config['redshift_port']}"
)
with psycopg2.connect(
connection_factory=MillisLoggingConnection,
host=config.get('redshift_host'),
port=config.get('redshift_port', 5439),
dbname=config.get('redshift_database'),
user=config.get('redshift_username'),
password=config.get('redshift_password')) as connection:
s3_config = config.get('target_s3')
s3 = S3(s3_config.get('aws_access_key_id'),
s3_config.get('aws_secret_access_key'),
s3_config.get('bucket'),
s3_config.get('key_prefix'),
aws_session_token=s3_config.get('aws_session_token'))
redshift_target = RedshiftTarget(
connection,
s3,
redshift_schema=config.get('redshift_schema', 'public'),
logging_level=config.get('logging_level'),
default_column_length=config.get('default_column_length',
1000),
persist_empty_tables=config.get('persist_empty_tables'))
if input_stream:
target_tools.stream_to_target(input_stream,
redshift_target,
config=config)
else:
target_tools.main(redshift_target)
finally:
if tunnel is not None:
tunnel.stop()
def main(config, input_stream=None):
with psycopg2.connect(
host=config.get('postgres_host', 'localhost'),
port=config.get('postgres_port', 5432),
dbname=config.get('postgres_database'),
user=config.get('postgres_username'),
password=config.get('postgres_password')) as connection:
postgres_target = PostgresTarget(connection,
postgres_schema=config.get(
'postgres_schema', 'public'))
if input_stream:
target_tools.stream_to_target(input_stream,
postgres_target,
config=config)
else:
target_tools.main(postgres_target)
def main(config, input_stream=None):
with psycopg2.connect(
connection_factory=MillisLoggingConnection,
host=config.get('postgres_host', 'localhost'),
port=config.get('postgres_port', 5432),
dbname=config.get('postgres_database'),
user=config.get('postgres_username'),
password=config.get('postgres_password')) as connection:
postgres_target = PostgresTarget(
connection,
postgres_schema=config.get('postgres_schema', 'public'),
logging_level=config.get('logging_level'),
persist_empty_tables=config.get('persist_empty_tables'))
if input_stream:
target_tools.stream_to_target(input_stream,
postgres_target,
config=config)
else:
target_tools.main(postgres_target)
