def main(config, input_stream=None):
with psycopg2.connect(
connection_factory=MillisLoggingConnection,
host=config.get('postgres_host', 'localhost'),
port=config.get('postgres_port', 5432),
dbname=config.get('postgres_database'),
user=config.get('postgres_username'),
password=config.get('postgres_password'),
sslmode=config.get('postgres_sslmode'),
sslcert=config.get('postgres_sslcert'),
sslkey=config.get('postgres_sslkey'),
sslrootcert=config.get('postgres_sslrootcert'),
sslcrl=config.get('postgres_sslcrl'),
application_name=config.get('application_name', 'target-postgres'),
) as connection:
postgres_target = PostgresTarget(
connection,
postgres_schema=config.get('postgres_schema', 'public'),
logging_level=config.get('logging_level'),
persist_empty_tables=config.get('persist_empty_tables'),
add_upsert_indexes=config.get('add_upsert_indexes', True),
before_run_sql=config.get('before_run_sql'),
after_run_sql=config.get('after_run_sql'),
)
if input_stream:
target_tools.stream_to_target(input_stream,
postgres_target,
config=config)
else:
target_tools.main(postgres_target)
def main(config, input_stream=None):
streams = {}
try:
if not config.get('disable_collection', False):
LOGGER.info('Sending version information to singer.io. ' +
'To disable sending anonymous usage data, set ' +
'the config parameter "disable_collection" to true')
threading.Thread(target=send_usage_stats).start()
connection = psycopg2.connect(host=config.get('postgres_host',
'localhost'),
port=config.get('postgres_port', 5432),
dbname=config.get('postgres_database'),
user=config.get('postgres_username'),
password=config.get('postgres_password'))
postgres_target = PostgresTarget(connection,
LOGGER,
postgres_schema=config.get(
'postgres_schema', 'public'))
invalid_records_detect = config.get('invalid_records_detect')
invalid_records_threshold = config.get('invalid_records_threshold')
max_batch_rows = config.get('max_batch_rows')
max_batch_size = config.get('max_batch_size')
batch_detection_threshold = config.get('batch_detection_threshold',
5000)
if not input_stream:
input_stream = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8')
line_count = 0
for line in input_stream:
line_handler(streams, postgres_target, invalid_records_detect,
invalid_records_threshold, max_batch_rows,
max_batch_size, line)
if line_count > 0 and line_count % batch_detection_threshold == 0:
flush_streams(streams, postgres_target)
line_count += 1
flush_streams(streams, postgres_target, force=True)
connection.close()
except Exception as e:
LOGGER.critical(e)
raise e
finally:
report_invalid_records(streams)
def main(config, input_stream=None):
with psycopg2.connect(
host=config.get('postgres_host', 'localhost'),
port=config.get('postgres_port', 5432),
dbname=config.get('postgres_database'),
user=config.get('postgres_username'),
password=config.get('postgres_password')) as connection:
postgres_target = PostgresTarget(connection,
postgres_schema=config.get(
'postgres_schema', 'public'))
if input_stream:
target_tools.stream_to_target(input_stream,
postgres_target,
config=config)
else:
target_tools.main(postgres_target)
def main(config, input_stream=None):
with psycopg2.connect(
connection_factory=MillisLoggingConnection,
host=config.get('postgres_host', 'localhost'),
port=config.get('postgres_port', 5432),
dbname=config.get('postgres_database'),
user=config.get('postgres_username'),
password=config.get('postgres_password')) as connection:
postgres_target = PostgresTarget(
connection,
postgres_schema=config.get('postgres_schema', 'public'),
logging_level=config.get('logging_level'),
persist_empty_tables=config.get('persist_empty_tables'))
if input_stream:
target_tools.stream_to_target(input_stream,
postgres_target,
config=config)
else:
target_tools.main(postgres_target)