def master(): db_conn = ibbdlib.get_db_conn(**db_server['223.4.155.172']) db_conn.execute("TRUNCATE TABLE ibbd2.lib_cat_industry_keyword_2") db_conn.close() ibbdlib.migratetable.migrate_table(db_server['223.4.155.152'], db_server['223.4.155.172'], 'ibbd2.user2') ibbdlib.migratetable.migrate_table(db_server['223.4.155.152'], db_server['223.4.155.172'], 'ibbd2.user_keywords') ibbdlib.migratetable.migrate_table(db_server['223.4.155.152'], db_server['223.4.155.172'], 'ibbd2.lib_cat_industry_keyword_2') ibbdlib.migratetable.migrate_table_by_query( db_server['223.4.155.152'], db_server['223.4.155.172'], 'ibbd2.user_shop_monitor_items', """SELECT user, shop_type, (CASE WHEN shop_id='' THEN 0 ELSE shop_id END) shop_id, user_num_id, item_id, status, population_tsmp FROM ibbd2.user_shop_monitor_items""") # keep connection dbConnPool = None reConn = None while True: try: dbConnPool = ConnectionPool(max_connections=1000, host='localhost', user='******', passwd='ibbd_etl_secure', db='topspider', charset='utf8') # reConn = getRedisConn2() reConn = ibbdlib.get_redis_conn(**redis_server) break except Exception as e: print e runKeywordsSchedule(reConn, dbConnPool) _ensure_schedule_complete(reConn) dbConnPool.disconnect() dbConnPool = ConnectionPool(max_connections=1000, host='localhost', user='******', passwd='ibbd_etl_secure', db='topspider', charset='utf8') while True: try: # migrateScheduleToRedis(reConn, dbConnPool) break except: traceback.print_exc() dbConnPool.disconnect() del reConn
def main(): connpool = ConnectionPool(max_connections=20, **{ 'host': 'localhost', 'user': '******', 'passwd': '', 'db': 'ibbd2', }) reConn = getRedisConn2() test_schedule_cat(reConn, connpool) del reConn connpool.disconnect()
def main(): _slave_info('initing...') # get slave ID global SLAVEID SLAVEID = getMacId() # get slave ID global IP IP = getMyIpInfo()['data']['ip'] # connection pool conn_config = slave_config.get('IP') or slave_config.get('default') conn_config = slave_config.get('default') dbConnPool = ConnectionPool(max_connections=1000, **conn_config.get('mysql')) reConn = getRedisConn2(conn_config.get('redis')) _slave_info('init... complete') _slave_info('slave start to work') dbConnPool.disconnect() del reConn
def master(): db_conn = ibbdlib.get_db_conn(**db_server['223.4.155.172']) db_conn.execute("TRUNCATE TABLE ibbd2.lib_cat_industry_keyword_2") db_conn.close() ibbdlib.migratetable.migrate_table( db_server['223.4.155.152'], db_server['223.4.155.172'], 'ibbd2.user2') ibbdlib.migratetable.migrate_table( db_server['223.4.155.152'], db_server['223.4.155.172'], 'ibbd2.user_keywords') ibbdlib.migratetable.migrate_table( db_server['223.4.155.152'], db_server['223.4.155.172'], 'ibbd2.lib_cat_industry_keyword_2') ibbdlib.migratetable.migrate_table_by_query( db_server['223.4.155.152'], db_server['223.4.155.172'], 'ibbd2.user_shop_monitor_items', """SELECT user, shop_type, (CASE WHEN shop_id='' THEN 0 ELSE shop_id END) shop_id, user_num_id, item_id, status, population_tsmp FROM ibbd2.user_shop_monitor_items""" ) # keep connection dbConnPool = None reConn = None while True: try: dbConnPool = ConnectionPool(max_connections=1000, host='localhost', user='******', passwd='ibbd_etl_secure', db='topspider', charset='utf8') # reConn = getRedisConn2() reConn = ibbdlib.get_redis_conn(**redis_server) break except Exception as e: print e runKeywordsSchedule(reConn, dbConnPool) _ensure_schedule_complete(reConn) dbConnPool.disconnect() dbConnPool = ConnectionPool(max_connections=1000, host='localhost', user='******', passwd='ibbd_etl_secure', db='topspider', charset='utf8') while True: try: # migrateScheduleToRedis(reConn, dbConnPool) break except: traceback.print_exc() dbConnPool.disconnect() del reConn