def main_impl(): args = utils.parse_args(REQUIRED_CONFIG_KEYS) connection = MySQLConnection(args.config) connect_with_backoff(connection) warnings = [] with connection.cursor() as cur: try: cur.execute('SET @@session.time_zone="+0:00"') except pymysql.err.InternalError as e: warnings.append( 'Could not set session.time_zone. Error: ({}) {}'.format( *e.args)) try: cur.execute('SET @@session.wait_timeout=2700') except pymysql.err.InternalError as e: warnings.append( 'Could not set session.wait_timeout. Error: ({}) {}'.format( *e.args)) try: cur.execute('SET @@session.innodb_lock_wait_timeout=2700') except pymysql.err.InternalError as e: warnings.append( 'Could not set session.innodb_lock_wait_timeout. Error: ({}) {}' .format(*e.args)) if warnings: LOGGER.info(( "Encountered non-fatal errors when configuring MySQL session that could " "impact performance:")) for w in warnings: LOGGER.warning(w) log_server_params(connection) if args.discover: do_discover(connection) elif args.catalog: state = args.state or {} do_sync(connection, args.config, args.catalog, state) elif args.properties: catalog = Catalog.from_dict(args.properties) state = args.state or {} do_sync(connection, args.config, catalog, state) else: LOGGER.info("No properties were selected")
def main(): args = utils.parse_args(REQUIRED_CONFIG_KEYS) #NB> this code will only work correctly when the local time is set to UTC because of calls to the timestamp() method. os.environ['TZ'] = 'UTC' # gcloud fails creating temporary tables if it is inside of transactions mysql_config = args.config mysql_config["autocommit"] = True mysql_conn = MySQLConnection(mysql_config) validate_only = args.config.get("validate_only") if not validate_only: log_server_params(mysql_conn) try: if validate_only: validate_connect(mysql_conn) elif args.discover: do_discover(mysql_conn, args.config) elif args.catalog: state = args.state or {} do_sync(mysql_conn, args.config, args.catalog, state) elif args.properties: catalog = Catalog.from_dict(args.properties) state = args.state or {} do_sync(mysql_conn, args.config, catalog, state) else: LOGGER.info("No properties were selected") except Exception as e: raise e
def get_test_connection(): db_config = get_db_config() con = pymysql.connect(**db_config) try: with con.cursor() as cur: try: cur.execute('DROP DATABASE {}'.format(DB_NAME)) except: pass cur.execute('CREATE DATABASE {}'.format(DB_NAME)) finally: con.close() db_config['database'] = DB_NAME db_config['autocommit'] = True mysql_conn = MySQLConnection(db_config) mysql_conn.autocommit_mode = True return mysql_conn
def get_test_connection(extra_config=None): db_config = get_db_config() con = pymysql.connect(**db_config) try: with con.cursor() as cur: try: cur.execute("DROP DATABASE {}".format(DB_NAME)) except: pass cur.execute("CREATE DATABASE {}".format(DB_NAME)) finally: con.close() db_config["database"] = DB_NAME db_config["autocommit"] = True if not extra_config: extra_config = {} mysql_conn = MySQLConnection({**db_config, **extra_config}) mysql_conn.autocommit_mode = True return mysql_conn
def main_impl(): args = utils.parse_args(REQUIRED_CONFIG_KEYS) mysql_conn = MySQLConnection(args.config) log_server_params(mysql_conn) if args.discover: do_discover(mysql_conn, args.config) elif args.catalog: state = args.state or {} do_sync(mysql_conn, args.config, args.catalog, state) elif args.properties: catalog = Catalog.from_dict(args.properties) state = args.state or {} do_sync(mysql_conn, args.config, catalog, state) else: LOGGER.info("No properties were selected")
def main_impl(): args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS) mysql_conn = MySQLConnection(args.config) log_server_params(mysql_conn) if args.discover: do_discover(mysql_conn, args.config) elif args.catalog: state = args.state or {} do_sync(mysql_conn, args.config, args.catalog, state) elif args.properties: catalog = Catalog.from_dict(args.properties) state = args.state or {} do_sync(mysql_conn, args.config, catalog, state) else: raise ValueError("Hmm I don't know what to do! Neither discovery nor sync mode was selected.")
def main(): args = utils.parse_args(REQUIRED_CONFIG_KEYS) #NB> this code will only work correctly when the local time is set to UTC because of calls to the timestamp() method. os.environ['TZ'] = 'UTC' mysql_conn = MySQLConnection(args.config) log_server_params(mysql_conn) if args.discover: do_discover(mysql_conn, args.config) elif args.catalog: state = args.state or {} do_sync(mysql_conn, args.config, args.catalog, state) elif args.properties: catalog = Catalog.from_dict(args.properties) state = args.state or {} do_sync(mysql_conn, args.config, catalog, state) else: LOGGER.info("No properties were selected")
def main_impl(): args = utils.parse_args(REQUIRED_CONFIG_KEYS, CONFIG_OBJ.items('tap_config')) timedelta = datetime.timedelta mysql_conn = MySQLConnection(args.config) log_server_params(mysql_conn) original_state_file = getattr(args, 'original_state_file', '') state = args.state or {} catalog = {} if args.discover: do_discover(mysql_conn, args.config) return elif args.catalog: catalog = args.catalog elif args.properties: catalog = Catalog.from_dict(args.properties) else: LOGGER.warning("No properties were selected") return # Start the syncing loop. if str(CONFIG_OBJ.get('tap_config', 'schedule_sync')).upper() == 'TRUE': interval = CONFIG_OBJ.get('tap_config', 'interval') runtime = CONFIG_OBJ.get('tap_config', 'runtime') # Set initial runtime stripping out seconds and microseconds. datetimeobj = datetime.datetime nextrundate = datetimeobj.now().strftime('%Y-%m-%d') nextrundatetime = datetimeobj.strptime( '%s %s' % (nextrundate, runtime), '%Y-%m-%d %H:%M:%S') nextrundatetime = nextrundatetime.replace(second=0, microsecond=0) logcount = 0 # Start the syncing loop. while True: # Run sync if next runtime is now. if nextrundatetime == datetimeobj.now().replace(second=0, microsecond=0): delete_old_logs() do_sync(mysql_conn, args.config, catalog, state, original_state_file) time.sleep(60) logcount = 0 # If next runtime is earlier then current time, update next runtime based on interval. elif nextrundatetime < datetimeobj.now(): if interval == 'hourly': nextrundatetime = nextrundatetime + timedelta(hours=1) elif interval == 'daily': nextrundatetime = nextrundatetime + timedelta(days=1) elif interval == 'weekly': nextrundatetime = nextrundatetime + timedelta(days=7) # Print what's happening. LOGGER.info( 'Waiting for next run time: %s \r' % datetimeobj.strftime(nextrundatetime, '%Y-%m-%d %H:%M:%S')) time.sleep(60) logcount += 1 # Print what's happening. elif logcount == 0: LOGGER.info( 'Waiting for next run time: %s \r' % datetimeobj.strftime(nextrundatetime, '%Y-%m-%d %H:%M:%S')) logcount += 1 # Not time to sync, continue waiting... else: time.sleep(60) else: delete_old_logs() do_sync(mysql_conn, args.config, catalog, state, original_state_file)