コード例 #1
0
ファイル: master.py プロジェクト: cash2one/dataMining-1
def master():
    db_conn = ibbdlib.get_db_conn(**db_server['223.4.155.172'])
    db_conn.execute("TRUNCATE TABLE ibbd2.lib_cat_industry_keyword_2")
    db_conn.close()
    ibbdlib.migratetable.migrate_table(db_server['223.4.155.152'],
                                       db_server['223.4.155.172'],
                                       'ibbd2.user2')
    ibbdlib.migratetable.migrate_table(db_server['223.4.155.152'],
                                       db_server['223.4.155.172'],
                                       'ibbd2.user_keywords')
    ibbdlib.migratetable.migrate_table(db_server['223.4.155.152'],
                                       db_server['223.4.155.172'],
                                       'ibbd2.lib_cat_industry_keyword_2')
    ibbdlib.migratetable.migrate_table_by_query(
        db_server['223.4.155.152'], db_server['223.4.155.172'],
        'ibbd2.user_shop_monitor_items', """SELECT user, shop_type,
        (CASE WHEN shop_id='' THEN 0 ELSE shop_id END) shop_id,
        user_num_id, item_id, status,
        population_tsmp
        FROM ibbd2.user_shop_monitor_items""")

    # keep connection

    dbConnPool = None
    reConn = None
    while True:
        try:
            dbConnPool = ConnectionPool(max_connections=1000,
                                        host='localhost',
                                        user='******',
                                        passwd='ibbd_etl_secure',
                                        db='topspider',
                                        charset='utf8')
            # reConn = getRedisConn2()
            reConn = ibbdlib.get_redis_conn(**redis_server)
            break
        except Exception as e:
            print e
    runKeywordsSchedule(reConn, dbConnPool)
    _ensure_schedule_complete(reConn)
    dbConnPool.disconnect()
    dbConnPool = ConnectionPool(max_connections=1000,
                                host='localhost',
                                user='******',
                                passwd='ibbd_etl_secure',
                                db='topspider',
                                charset='utf8')
    while True:
        try:
            # migrateScheduleToRedis(reConn, dbConnPool)
            break
        except:
            traceback.print_exc()
    dbConnPool.disconnect()
    del reConn
コード例 #2
0
def main():
    connpool = ConnectionPool(max_connections=20, **{
        'host': 'localhost',
        'user': '******',
        'passwd': '',
        'db': 'ibbd2',
        })
    reConn = getRedisConn2()
    test_schedule_cat(reConn, connpool)
    del reConn
    connpool.disconnect()
コード例 #3
0
def main():
    _slave_info('initing...')

    # get slave ID

    global SLAVEID
    SLAVEID = getMacId()

    # get slave ID

    global IP
    IP = getMyIpInfo()['data']['ip']

    # connection pool

    conn_config = slave_config.get('IP') or slave_config.get('default')
    conn_config = slave_config.get('default')
    dbConnPool = ConnectionPool(max_connections=1000,
                                **conn_config.get('mysql'))
    reConn = getRedisConn2(conn_config.get('redis'))
    _slave_info('init... complete')
    _slave_info('slave start to work')
    dbConnPool.disconnect()
    del reConn
コード例 #4
0
ファイル: master.py プロジェクト: ruige123456/dataMining
def master():
    db_conn = ibbdlib.get_db_conn(**db_server['223.4.155.172'])
    db_conn.execute("TRUNCATE TABLE ibbd2.lib_cat_industry_keyword_2")
    db_conn.close()
    ibbdlib.migratetable.migrate_table(
        db_server['223.4.155.152'],
        db_server['223.4.155.172'],
        'ibbd2.user2')
    ibbdlib.migratetable.migrate_table(
        db_server['223.4.155.152'],
        db_server['223.4.155.172'],
        'ibbd2.user_keywords')
    ibbdlib.migratetable.migrate_table(
        db_server['223.4.155.152'],
        db_server['223.4.155.172'],
        'ibbd2.lib_cat_industry_keyword_2')
    ibbdlib.migratetable.migrate_table_by_query(
        db_server['223.4.155.152'],
        db_server['223.4.155.172'],
        'ibbd2.user_shop_monitor_items',
        """SELECT user, shop_type,
        (CASE WHEN shop_id='' THEN 0 ELSE shop_id END) shop_id,
        user_num_id, item_id, status,
        population_tsmp
        FROM ibbd2.user_shop_monitor_items"""
    )

    # keep connection

    dbConnPool = None
    reConn = None
    while True:
        try:
            dbConnPool = ConnectionPool(max_connections=1000,
                                        host='localhost',
                                        user='******',
                                        passwd='ibbd_etl_secure',
                                        db='topspider',
                                        charset='utf8')
            # reConn = getRedisConn2()
            reConn = ibbdlib.get_redis_conn(**redis_server)
            break
        except Exception as e:
            print e
    runKeywordsSchedule(reConn, dbConnPool)
    _ensure_schedule_complete(reConn)
    dbConnPool.disconnect()
    dbConnPool = ConnectionPool(max_connections=1000,
                                host='localhost',
                                user='******',
                                passwd='ibbd_etl_secure',
                                db='topspider',
                                charset='utf8')
    while True:
        try:
            # migrateScheduleToRedis(reConn, dbConnPool)
            break
        except:
            traceback.print_exc()
    dbConnPool.disconnect()
    del reConn