def main_impl():
args = utils.parse_args(REQUIRED_CONFIG_KEYS)
connection = MySQLConnection(args.config)
connect_with_backoff(connection)
warnings = []
with connection.cursor() as cur:
try:
cur.execute('SET @@session.time_zone="+0:00"')
except pymysql.err.InternalError as e:
warnings.append(
'Could not set session.time_zone. Error: ({}) {}'.format(
*e.args))
try:
cur.execute('SET @@session.wait_timeout=2700')
except pymysql.err.InternalError as e:
warnings.append(
'Could not set session.wait_timeout. Error: ({}) {}'.format(
*e.args))
try:
cur.execute('SET @@session.innodb_lock_wait_timeout=2700')
except pymysql.err.InternalError as e:
warnings.append(
'Could not set session.innodb_lock_wait_timeout. Error: ({}) {}'
.format(*e.args))
if warnings:
LOGGER.info((
"Encountered non-fatal errors when configuring MySQL session that could "
"impact performance:"))
for w in warnings:
LOGGER.warning(w)
log_server_params(connection)
if args.discover:
do_discover(connection)
elif args.catalog:
state = args.state or {}
do_sync(connection, args.config, args.catalog, state)
elif args.properties:
catalog = Catalog.from_dict(args.properties)
state = args.state or {}
do_sync(connection, args.config, catalog, state)
else:
LOGGER.info("No properties were selected")
def main():
args = utils.parse_args(REQUIRED_CONFIG_KEYS)
#NB> this code will only work correctly when the local time is set to UTC because of calls to the timestamp() method.
os.environ['TZ'] = 'UTC'
# gcloud fails creating temporary tables if it is inside of transactions
mysql_config = args.config
mysql_config["autocommit"] = True
mysql_conn = MySQLConnection(mysql_config)
validate_only = args.config.get("validate_only")
if not validate_only:
log_server_params(mysql_conn)
try:
if validate_only:
validate_connect(mysql_conn)
elif args.discover:
do_discover(mysql_conn, args.config)
elif args.catalog:
state = args.state or {}
do_sync(mysql_conn, args.config, args.catalog, state)
elif args.properties:
catalog = Catalog.from_dict(args.properties)
state = args.state or {}
do_sync(mysql_conn, args.config, catalog, state)
else:
LOGGER.info("No properties were selected")
except Exception as e:
raise e
def get_test_connection():
db_config = get_db_config()
con = pymysql.connect(**db_config)
try:
with con.cursor() as cur:
try:
cur.execute('DROP DATABASE {}'.format(DB_NAME))
except:
pass
cur.execute('CREATE DATABASE {}'.format(DB_NAME))
finally:
con.close()
db_config['database'] = DB_NAME
db_config['autocommit'] = True
mysql_conn = MySQLConnection(db_config)
mysql_conn.autocommit_mode = True
return mysql_conn
def get_test_connection(extra_config=None):
db_config = get_db_config()
con = pymysql.connect(**db_config)
try:
with con.cursor() as cur:
try:
cur.execute("DROP DATABASE {}".format(DB_NAME))
except:
pass
cur.execute("CREATE DATABASE {}".format(DB_NAME))
finally:
con.close()
db_config["database"] = DB_NAME
db_config["autocommit"] = True
if not extra_config:
extra_config = {}
mysql_conn = MySQLConnection({**db_config, **extra_config})
mysql_conn.autocommit_mode = True
return mysql_conn
def main_impl():
args = utils.parse_args(REQUIRED_CONFIG_KEYS)
mysql_conn = MySQLConnection(args.config)
log_server_params(mysql_conn)
if args.discover:
do_discover(mysql_conn, args.config)
elif args.catalog:
state = args.state or {}
do_sync(mysql_conn, args.config, args.catalog, state)
elif args.properties:
catalog = Catalog.from_dict(args.properties)
state = args.state or {}
do_sync(mysql_conn, args.config, catalog, state)
else:
LOGGER.info("No properties were selected")
def main_impl():
args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS)
mysql_conn = MySQLConnection(args.config)
log_server_params(mysql_conn)
if args.discover:
do_discover(mysql_conn, args.config)
elif args.catalog:
state = args.state or {}
do_sync(mysql_conn, args.config, args.catalog, state)
elif args.properties:
catalog = Catalog.from_dict(args.properties)
state = args.state or {}
do_sync(mysql_conn, args.config, catalog, state)
else:
raise ValueError("Hmm I don't know what to do! Neither discovery nor sync mode was selected.")
def main():
args = utils.parse_args(REQUIRED_CONFIG_KEYS)
#NB> this code will only work correctly when the local time is set to UTC because of calls to the timestamp() method.
os.environ['TZ'] = 'UTC'
mysql_conn = MySQLConnection(args.config)
log_server_params(mysql_conn)
if args.discover:
do_discover(mysql_conn, args.config)
elif args.catalog:
state = args.state or {}
do_sync(mysql_conn, args.config, args.catalog, state)
elif args.properties:
catalog = Catalog.from_dict(args.properties)
state = args.state or {}
do_sync(mysql_conn, args.config, catalog, state)
else:
LOGGER.info("No properties were selected")
def main_impl():
args = utils.parse_args(REQUIRED_CONFIG_KEYS,
CONFIG_OBJ.items('tap_config'))
timedelta = datetime.timedelta
mysql_conn = MySQLConnection(args.config)
log_server_params(mysql_conn)
original_state_file = getattr(args, 'original_state_file', '')
state = args.state or {}
catalog = {}
if args.discover:
do_discover(mysql_conn, args.config)
return
elif args.catalog:
catalog = args.catalog
elif args.properties:
catalog = Catalog.from_dict(args.properties)
else:
LOGGER.warning("No properties were selected")
return
# Start the syncing loop.
if str(CONFIG_OBJ.get('tap_config', 'schedule_sync')).upper() == 'TRUE':
interval = CONFIG_OBJ.get('tap_config', 'interval')
runtime = CONFIG_OBJ.get('tap_config', 'runtime')
# Set initial runtime stripping out seconds and microseconds.
datetimeobj = datetime.datetime
nextrundate = datetimeobj.now().strftime('%Y-%m-%d')
nextrundatetime = datetimeobj.strptime(
'%s %s' % (nextrundate, runtime), '%Y-%m-%d %H:%M:%S')
nextrundatetime = nextrundatetime.replace(second=0, microsecond=0)
logcount = 0
# Start the syncing loop.
while True:
# Run sync if next runtime is now.
if nextrundatetime == datetimeobj.now().replace(second=0,
microsecond=0):
delete_old_logs()
do_sync(mysql_conn, args.config, catalog, state,
original_state_file)
time.sleep(60)
logcount = 0
# If next runtime is earlier then current time, update next runtime based on interval.
elif nextrundatetime < datetimeobj.now():
if interval == 'hourly':
nextrundatetime = nextrundatetime + timedelta(hours=1)
elif interval == 'daily':
nextrundatetime = nextrundatetime + timedelta(days=1)
elif interval == 'weekly':
nextrundatetime = nextrundatetime + timedelta(days=7)
# Print what's happening.
LOGGER.info(
'Waiting for next run time: %s \r' %
datetimeobj.strftime(nextrundatetime, '%Y-%m-%d %H:%M:%S'))
time.sleep(60)
logcount += 1
# Print what's happening.
elif logcount == 0:
LOGGER.info(
'Waiting for next run time: %s \r' %
datetimeobj.strftime(nextrundatetime, '%Y-%m-%d %H:%M:%S'))
logcount += 1
# Not time to sync, continue waiting...
else:
time.sleep(60)
else:
delete_old_logs()
do_sync(mysql_conn, args.config, catalog, state, original_state_file)
