Ejemplo n.º 1
0
 def test_getitemsbykeyword(self):
     for keyword in ['FANCL 乳液']:
         metadata = getItemsByKeyword(keyword,
                                      'utf-8',
                                      page=1,
                                      sort='sale-desc')
         print json.dumps(metadata, indent=4, ensure_ascii=False)
Ejemplo n.º 2
0
 def test_getitemsbykeyword(self):
     for keyword in [
             '山地车',
             # 'i9300'
     ]:
         metadata = getItemsByKeyword(keyword, 'utf-8', page=1)
         for i, item in enumerate(metadata):
             print json.dumps(item, indent=4)
Ejemplo n.º 3
0
 def test_getitemsbykeyword(self):
     for keyword in [
         '山地车',
         # 'i9300'
     ]:
         metadata = getItemsByKeyword(keyword, 'utf-8', page=1)
         for i, item in enumerate(metadata):
             print json.dumps(item, indent=4)
Ejemplo n.º 4
0
    FROM(
        SELECT DISTINCT(keyword) AS keyword
        FROM ibbd2.user_keywords T3
        WHERE T3.status='1'
        ORDER BY keyword
    ) T2
    LEFT JOIN(
        SELECT DISTINCT(keyword) AS keyword
        FROM topspider.top_itemsearchresult T1
        WHERE T1.population_tsmp > curdate()
    )T4
    ON T2.keyword = T4.keyword
    WHERE T4.keyword IS NULL"""
for row in db_conn.iter(query):
    keyword = row.keyword.encode('utf-8')
    items = getItemsByKeyword(keyword, 'utf-8', page=5)
    saveKeywordSearchResult(items, db_pool)
    time.sleep(1)
db_conn.close()
# db_pool.disconnect()

re_conn = ibbdlib.get_redis_conn(**{
    'host': 'localhost',
    'port': 6379,
    'password': '******'
})
db_conn = ibbdlib.get_db_conn(**db_server)
iids = set()
uids = set()
query = """SELECT DISTINCT item_id, user_num_id
FROM topspider.top_itemsearchresult
Ejemplo n.º 5
0
def executeKeywordSearchResult(
    reConn,
    dbConnPool,
    schedule_name='Schedule_keywordSearchResult',
    schedule_error_name='Schedule_keywordSearchResultError',
    schedule_process_name='Schedule_keywordSearchResult_process',
    schedule_config_name='Schedule_keywordSearchResult_config',
    ):
    """执行关键词更新任务
    
    根据提供的schedele_name等获取任务信息和配置,执行任务将结果保存到数据库
    
    Args:
        reConn  Redis连接
        dbConnPool  数据库连接池
        schedule_*  Redis中的schedule配置信息键名
    
    """

    n = 0
    for i in range(50):

        # slave config

        _slave_interval(reConn, n)

        # pipe

        keyword = reConn.spop(schedule_name)
        if keyword:
            keyword = unquote(keyword)
            keywordconfig = json.loads(reConn.hget(schedule_config_name, quote(keyword)))

            # end pipe

            if keyword:
                t1 = t2 = time.time()
                _slave_info(schedule_name, 'Start', quote(keyword))
                reConn.hset(schedule_process_name, quote(keyword), 1)
                try:
                    metadata = getItemsByKeyword(keyword, 'utf-8', keywordconfig.get('page'), keywordconfig.get('price'
                                                 ))
                    t2 = time.time()

                    threading.Thread(target=_thread_top_bll, args=(
                        saveKeywordSearchResult,
                        metadata,
                        dbConnPool['master'],
                        reConn,
                        schedule_name,
                        quote(keyword),
                        schedule_process_name,
                        schedule_error_name,
                        t1,
                        t2,
                        )).start()
                except Exception, e:
                    _slave_error(
                        schedule_name,
                        'Error',
                        quote(keyword),
                        str(e),
                        'req',
                        '%.1f' % (t2 - t1),
                        'save',
                        '%.1f' % (time.time() - t2),
                        )
                    reConn.hset(schedule_error_name, quote(keyword), json.dumps({'SlaveID': SLAVEID, 'msg': str(e)}))
        else:
            break
Ejemplo n.º 6
0
 def test_getitemsbykeyword(self):
     for keyword in [
         'FANCL 乳液'
     ]:
         metadata = getItemsByKeyword(keyword, 'utf-8', page=1, sort='sale-desc')
         print json.dumps(metadata, indent=4, ensure_ascii=False)
Ejemplo n.º 7
0
def executeKeywordSearchResult(
    schedule_name='Schedule_keywordSearchResult',
    schedule_error_name='Schedule_keywordSearchResultError',
    schedule_process_name='Schedule_keywordSearchResult_process',
    schedule_config_name='Schedule_keywordSearchResult_config',
):
    """执行关键词更新任务

    根据提供的schedele_name等获取任务信息和配置,执行任务将结果保存到数据库

    Args:
        schedule_*  Redis中的schedule配置信息键名

    """

    global RE_CONN, INTERRUPT
    dbConnPool = _reload_slave_db_pool(schedule_name)
    n = 0
    for i in range(50):
        if INTERRUPT:
            INTERRUPT = False
            break

        # slave config
        _slave_interval(n)

        keyword = RE_CONN.spop(schedule_name)
        if keyword:
            keyword = unquote(keyword)
            keywordconfig = json.loads(
                RE_CONN.hget(schedule_config_name, quote(keyword)))

            if keyword:
                t1 = t2 = time.time()
                _slave_info(schedule_name, 'Start', quote(keyword))
                RE_CONN.hset(schedule_process_name, quote(keyword), 1)
                try:
                    metadata = getItemsByKeyword(keyword, 'utf-8',
                                                 keywordconfig.get('page'),
                                                 keywordconfig.get('price'))
                    if len(metadata) == 0:
                        raise Exception('items not found')
                    t2 = time.time()

                    bll_queue.put(
                        (saveKeywordSearchResult, metadata, dbConnPool, None,
                         schedule_name, quote(keyword), schedule_process_name,
                         schedule_error_name, t1, t2))
                except:
                    _slave_error(
                        schedule_name,
                        'Error',
                        quote(keyword),
                        traceback.format_exc(),
                        'req',
                        '%.1f' % (t2 - t1),
                        'save',
                        '%.1f' % (time.time() - t2),
                    )
                    RE_CONN.hset(
                        schedule_error_name, quote(keyword),
                        json.dumps({
                            'SlaveID': SLAVEID,
                            'msg': traceback.format_exc()
                        }))
        else:
            break
Ejemplo n.º 8
0
    FROM(
        SELECT DISTINCT(keyword) AS keyword
        FROM ibbd2.user_keywords T3
        WHERE T3.status='1'
        ORDER BY keyword
    ) T2
    LEFT JOIN(
        SELECT DISTINCT(keyword) AS keyword
        FROM topspider.top_itemsearchresult T1
        WHERE T1.population_tsmp > curdate()
    )T4
    ON T2.keyword = T4.keyword
    WHERE T4.keyword IS NULL"""
for row in db_conn.iter(query):
    keyword = row.keyword.encode('utf-8')
    items = getItemsByKeyword(keyword, 'utf-8', page=5)
    saveKeywordSearchResult(items, db_pool)
    time.sleep(1)
db_conn.close()
# db_pool.disconnect()


re_conn = ibbdlib.get_redis_conn(**{
    'host': 'localhost',
    'port': 6379,
    'password': '******'
})
db_conn = ibbdlib.get_db_conn(**db_server)
iids = set()
uids = set()
query = """SELECT DISTINCT item_id, user_num_id