def test_getitemsbykeyword(self): for keyword in ['FANCL 乳液']: metadata = getItemsByKeyword(keyword, 'utf-8', page=1, sort='sale-desc') print json.dumps(metadata, indent=4, ensure_ascii=False)
def test_getitemsbykeyword(self): for keyword in [ '山地车', # 'i9300' ]: metadata = getItemsByKeyword(keyword, 'utf-8', page=1) for i, item in enumerate(metadata): print json.dumps(item, indent=4)
FROM( SELECT DISTINCT(keyword) AS keyword FROM ibbd2.user_keywords T3 WHERE T3.status='1' ORDER BY keyword ) T2 LEFT JOIN( SELECT DISTINCT(keyword) AS keyword FROM topspider.top_itemsearchresult T1 WHERE T1.population_tsmp > curdate() )T4 ON T2.keyword = T4.keyword WHERE T4.keyword IS NULL""" for row in db_conn.iter(query): keyword = row.keyword.encode('utf-8') items = getItemsByKeyword(keyword, 'utf-8', page=5) saveKeywordSearchResult(items, db_pool) time.sleep(1) db_conn.close() # db_pool.disconnect() re_conn = ibbdlib.get_redis_conn(**{ 'host': 'localhost', 'port': 6379, 'password': '******' }) db_conn = ibbdlib.get_db_conn(**db_server) iids = set() uids = set() query = """SELECT DISTINCT item_id, user_num_id FROM topspider.top_itemsearchresult
def executeKeywordSearchResult( reConn, dbConnPool, schedule_name='Schedule_keywordSearchResult', schedule_error_name='Schedule_keywordSearchResultError', schedule_process_name='Schedule_keywordSearchResult_process', schedule_config_name='Schedule_keywordSearchResult_config', ): """执行关键词更新任务 根据提供的schedele_name等获取任务信息和配置,执行任务将结果保存到数据库 Args: reConn Redis连接 dbConnPool 数据库连接池 schedule_* Redis中的schedule配置信息键名 """ n = 0 for i in range(50): # slave config _slave_interval(reConn, n) # pipe keyword = reConn.spop(schedule_name) if keyword: keyword = unquote(keyword) keywordconfig = json.loads(reConn.hget(schedule_config_name, quote(keyword))) # end pipe if keyword: t1 = t2 = time.time() _slave_info(schedule_name, 'Start', quote(keyword)) reConn.hset(schedule_process_name, quote(keyword), 1) try: metadata = getItemsByKeyword(keyword, 'utf-8', keywordconfig.get('page'), keywordconfig.get('price' )) t2 = time.time() threading.Thread(target=_thread_top_bll, args=( saveKeywordSearchResult, metadata, dbConnPool['master'], reConn, schedule_name, quote(keyword), schedule_process_name, schedule_error_name, t1, t2, )).start() except Exception, e: _slave_error( schedule_name, 'Error', quote(keyword), str(e), 'req', '%.1f' % (t2 - t1), 'save', '%.1f' % (time.time() - t2), ) reConn.hset(schedule_error_name, quote(keyword), json.dumps({'SlaveID': SLAVEID, 'msg': str(e)})) else: break
def test_getitemsbykeyword(self): for keyword in [ 'FANCL 乳液' ]: metadata = getItemsByKeyword(keyword, 'utf-8', page=1, sort='sale-desc') print json.dumps(metadata, indent=4, ensure_ascii=False)
def executeKeywordSearchResult( schedule_name='Schedule_keywordSearchResult', schedule_error_name='Schedule_keywordSearchResultError', schedule_process_name='Schedule_keywordSearchResult_process', schedule_config_name='Schedule_keywordSearchResult_config', ): """执行关键词更新任务 根据提供的schedele_name等获取任务信息和配置,执行任务将结果保存到数据库 Args: schedule_* Redis中的schedule配置信息键名 """ global RE_CONN, INTERRUPT dbConnPool = _reload_slave_db_pool(schedule_name) n = 0 for i in range(50): if INTERRUPT: INTERRUPT = False break # slave config _slave_interval(n) keyword = RE_CONN.spop(schedule_name) if keyword: keyword = unquote(keyword) keywordconfig = json.loads( RE_CONN.hget(schedule_config_name, quote(keyword))) if keyword: t1 = t2 = time.time() _slave_info(schedule_name, 'Start', quote(keyword)) RE_CONN.hset(schedule_process_name, quote(keyword), 1) try: metadata = getItemsByKeyword(keyword, 'utf-8', keywordconfig.get('page'), keywordconfig.get('price')) if len(metadata) == 0: raise Exception('items not found') t2 = time.time() bll_queue.put( (saveKeywordSearchResult, metadata, dbConnPool, None, schedule_name, quote(keyword), schedule_process_name, schedule_error_name, t1, t2)) except: _slave_error( schedule_name, 'Error', quote(keyword), traceback.format_exc(), 'req', '%.1f' % (t2 - t1), 'save', '%.1f' % (time.time() - t2), ) RE_CONN.hset( schedule_error_name, quote(keyword), json.dumps({ 'SlaveID': SLAVEID, 'msg': traceback.format_exc() })) else: break
FROM( SELECT DISTINCT(keyword) AS keyword FROM ibbd2.user_keywords T3 WHERE T3.status='1' ORDER BY keyword ) T2 LEFT JOIN( SELECT DISTINCT(keyword) AS keyword FROM topspider.top_itemsearchresult T1 WHERE T1.population_tsmp > curdate() )T4 ON T2.keyword = T4.keyword WHERE T4.keyword IS NULL""" for row in db_conn.iter(query): keyword = row.keyword.encode('utf-8') items = getItemsByKeyword(keyword, 'utf-8', page=5) saveKeywordSearchResult(items, db_pool) time.sleep(1) db_conn.close() # db_pool.disconnect() re_conn = ibbdlib.get_redis_conn(**{ 'host': 'localhost', 'port': 6379, 'password': '******' }) db_conn = ibbdlib.get_db_conn(**db_server) iids = set() uids = set() query = """SELECT DISTINCT item_id, user_num_id