def dump_all_index():
#es_db.es_init(seller_mapping, get_index_name(), const.type_seller)
#log.log(u"先清理之前遗留的数据")
#delete_all_index()
dump_time = datetime.datetime.now()
dump_worth_index()
#log.log(dump_time.strftime('%Y-%m-%d %X')+u'完成用户信息更新')
search_config.write_dump_time(const.type_worth, dump_time)
def dump_all_index():
#es_db.es_init(seller_mapping, get_index_name(), const.type_seller)
#log.log(u"先清理之前遗留的数据")
#delete_all_index()
from_time = '1970-01-01'
dump_time = datetime.datetime.now()
dump_demand_index(get_select_all_demand_sql())
search_config.write_dump_time(const.type_demand, dump_time)
def dump_all_index():
#es_db.es_init(seller_mapping, get_index_name(), const.type_seller)
from_time = '1970-01-01'
dump_time = datetime.datetime.now()
sql = 'SELECT id, merchant_id, name, characteristic, is_delivery+0, is_return+0, ' \
' online_status, visit_count FROM product WHERE online_status != 3; '
dump_product_index(sql)
search_config.write_dump_time(const.type_product, dump_time)
def dump_all_index():
#es_db.es_init(seller_mapping, get_index_name(), const.type_seller)
from_time = '1970-01-01'
dump_time = datetime.datetime.now()
sql = 'SELECT id, `name`, user_id, province_code, city_code, area_code, addr_country, lat, lon, ' \
' `desc`, `status`, visit_count, pay_status, is_support_credit+0, create_time, pac_set_id FROM merchant; '
dump_seller_index(sql)
search_config.write_dump_time(const.type_seller, dump_time)
def dump_all_index():
es_db.es_init(user_mapping, const.index_name, const.type_user)
from_time = '1970-01-01'
dump_time = datetime.datetime.now()
sql = DUMP_USER_SQL + ' where update_time BETWEEN "%s" AND "%s"' % (
from_time, dump_time)
print sql
dump_user_index(sql)
search_config.write_dump_time('user', dump_time)
pass
def dump_increment_index():
from_time = search_config.read_last_dump_time('user')
dump_time = datetime.datetime.now()
sql = DUMP_USER_SQL + ' where status = 0 and update_time BETWEEN "%s" AND "%s"' % (
from_time, dump_time)
print sql
dump_user_index(sql)
sql = DUMP_USER_SQL + ' where id in (select DISTINCT user_id from order_info where pay_receive_time ' \
'BETWEEN "%s" and "%s")' % (from_time, dump_time)
print sql
dump_user_index(sql)
## 更新user关联购物车和未支付订单商品ID
dump_user_cart_unpaid_products(from_time, dump_time)
delete_users_index()
search_config.write_dump_time('user', dump_time)
pass
def dump_user_profile(dumpall=None):
nullValue = '\\N'
def _put_value(jsonMap, key, value):
if value == nullValue:
return
jsonMap[key] = value
if dumpall:
start_timestamp = 0
end_timestamp = int(time.time())
else:
start_date = datetime.datetime.strptime(
search_config.read_last_dump_time('user_profile'), "%Y-%m-%d %X")
end_date = start_date + datetime.timedelta(days=1)
search_config.write_dump_time('user_profile', end_date)
start_timestamp = int(time.mktime(start_date.timetuple()))
end_timestamp = int(time.mktime(end_date.timetuple()))
log.log('开始导入用户画像(属性)：品牌、类目、功效')
sql = 'select user_id,' \
'brand_id_order,' \
'brand_id_cart,' \
'brand_id_collect,' \
'category_id_order,' \
'category_id_cart,' \
'category_id_collect,' \
'efficacy_id_order,' \
'efficacy_id_cart,' \
'efficacy_id_collect ' \
'from actionlog.user_profile ' \
'where create_update_time >= %(start_timestamp)s ' \
'and create_update_time < %(end_timestamp)s'
cur = es_db.conn_db2.cursor()
params = {
'start_timestamp': start_timestamp,
'end_timestamp': end_timestamp
}
log.log(sql)
log.log('statement parameters:%s' % str(params))
count = cur.execute(sql, params)
log.log('there has %s rows user record' % count)
index = 0
for result in cur.fetchall():
index += 1
brands = {}
categorys = {}
efficacys = {}
user_id = result[0]
try:
user = conn_es.get(const.index_name, const.type_user, user_id)
_put_value(brands, 'byOrder', result[1]) # 订单
_put_value(brands, 'byCart', result[2]) # 购物车
_put_value(brands, 'byCollect', result[3]) # 收藏
_put_value(categorys, 'byOrder', result[4]) # 订单
_put_value(categorys, 'byCart', result[5]) # 购物车
_put_value(categorys, 'byCollect', result[6]) # 收藏
_put_value(efficacys, 'byOrder', result[7]) # 订单
_put_value(efficacys, 'byCart', result[8]) # 购物车
_put_value(efficacys, 'byCollect', result[9]) # 收藏
user["PROFBrands"] = brands
user["PROFCategorys"] = categorys
user["PROFEfficacys"] = efficacys
conn_es.update(const.index_name,
const.type_user,
user_id,
document=user,
bulk=True)
except Exception, e:
log.log('error at dump_user_profile, user id is %d\n%s' %
(user_id, e))
if index % conn_es.bulk_size == 0 or index % count == 0:
log.log('已完成:' + '%.2f' % (index / count * 100) + "%")
def dump_all_index():
log.log(u'开始清除遗留的index数据')
dump_time = datetime.datetime.now()
dump_article_index(get_select_all_article_sql())
search_config.write_dump_time(const.type_article, dump_time)