def master():
db_conn = ibbdlib.get_db_conn(**db_server['223.4.155.172'])
db_conn.execute("TRUNCATE TABLE ibbd2.lib_cat_industry_keyword_2")
db_conn.close()
ibbdlib.migratetable.migrate_table(db_server['223.4.155.152'],
db_server['223.4.155.172'],
'ibbd2.user2')
ibbdlib.migratetable.migrate_table(db_server['223.4.155.152'],
db_server['223.4.155.172'],
'ibbd2.user_keywords')
ibbdlib.migratetable.migrate_table(db_server['223.4.155.152'],
db_server['223.4.155.172'],
'ibbd2.lib_cat_industry_keyword_2')
ibbdlib.migratetable.migrate_table_by_query(
db_server['223.4.155.152'], db_server['223.4.155.172'],
'ibbd2.user_shop_monitor_items', """SELECT user, shop_type,
(CASE WHEN shop_id='' THEN 0 ELSE shop_id END) shop_id,
user_num_id, item_id, status,
population_tsmp
FROM ibbd2.user_shop_monitor_items""")
# keep connection
dbConnPool = None
reConn = None
while True:
try:
dbConnPool = ConnectionPool(max_connections=1000,
host='localhost',
user='******',
passwd='ibbd_etl_secure',
db='topspider',
charset='utf8')
# reConn = getRedisConn2()
reConn = ibbdlib.get_redis_conn(**redis_server)
break
except Exception as e:
print e
runKeywordsSchedule(reConn, dbConnPool)
_ensure_schedule_complete(reConn)
dbConnPool.disconnect()
dbConnPool = ConnectionPool(max_connections=1000,
host='localhost',
user='******',
passwd='ibbd_etl_secure',
db='topspider',
charset='utf8')
while True:
try:
# migrateScheduleToRedis(reConn, dbConnPool)
break
except:
traceback.print_exc()
dbConnPool.disconnect()
del reConn
def main():
connpool = ConnectionPool(max_connections=20, **{
'host': 'localhost',
'user': '******',
'passwd': '',
'db': 'ibbd2',
})
reConn = getRedisConn2()
test_schedule_cat(reConn, connpool)
del reConn
connpool.disconnect()
def main():
_slave_info('initing...')
# get slave ID
global SLAVEID
SLAVEID = getMacId()
# get slave ID
global IP
IP = getMyIpInfo()['data']['ip']
# connection pool
conn_config = slave_config.get('IP') or slave_config.get('default')
conn_config = slave_config.get('default')
dbConnPool = ConnectionPool(max_connections=1000,
**conn_config.get('mysql'))
reConn = getRedisConn2(conn_config.get('redis'))
_slave_info('init... complete')
_slave_info('slave start to work')
dbConnPool.disconnect()
del reConn
def master():
db_conn = ibbdlib.get_db_conn(**db_server['223.4.155.172'])
db_conn.execute("TRUNCATE TABLE ibbd2.lib_cat_industry_keyword_2")
db_conn.close()
ibbdlib.migratetable.migrate_table(
db_server['223.4.155.152'],
db_server['223.4.155.172'],
'ibbd2.user2')
ibbdlib.migratetable.migrate_table(
db_server['223.4.155.152'],
db_server['223.4.155.172'],
'ibbd2.user_keywords')
ibbdlib.migratetable.migrate_table(
db_server['223.4.155.152'],
db_server['223.4.155.172'],
'ibbd2.lib_cat_industry_keyword_2')
ibbdlib.migratetable.migrate_table_by_query(
db_server['223.4.155.152'],
db_server['223.4.155.172'],
'ibbd2.user_shop_monitor_items',
"""SELECT user, shop_type,
(CASE WHEN shop_id='' THEN 0 ELSE shop_id END) shop_id,
user_num_id, item_id, status,
population_tsmp
FROM ibbd2.user_shop_monitor_items"""
)
# keep connection
dbConnPool = None
reConn = None
while True:
try:
dbConnPool = ConnectionPool(max_connections=1000,
host='localhost',
user='******',
passwd='ibbd_etl_secure',
db='topspider',
charset='utf8')
# reConn = getRedisConn2()
reConn = ibbdlib.get_redis_conn(**redis_server)
break
except Exception as e:
print e
runKeywordsSchedule(reConn, dbConnPool)
_ensure_schedule_complete(reConn)
dbConnPool.disconnect()
dbConnPool = ConnectionPool(max_connections=1000,
host='localhost',
user='******',
passwd='ibbd_etl_secure',
db='topspider',
charset='utf8')
while True:
try:
# migrateScheduleToRedis(reConn, dbConnPool)
break
except:
traceback.print_exc()
dbConnPool.disconnect()
del reConn
