def main(es):
"""
update all user in a day
"""
index_name = "20130903"
#index_name = time.strftime("%Y%m%d", time.localtime(time.time()-86400))
bool = es.indices.exists(index=index_name)
print bool
if not bool:
print "no index exist"
sys.exit(0)
user_rank = 0
bulk_action = []
n_range = range(0, 100000, 10000)
tb = time.time()
for left_range in n_range:
result = search_rank(index_name, left_range, 10000)
for item in result:
update_info = {}
user_rank += 1
update_info["user"] = item["_id"]
update_info["rank"] = user_rank
x = update_index_action(update_info, "rank", user_rank)
bulk_action.extend((x[0], x[1]))
if user_rank % 1000 == 0:
while 1:
try:
es.bulk(bulk_action,
index=index_name,
doc_type="bci",
timeout=30)
bulk_action = []
break
except Exception, r:
es = ES_CLUSTER_FLOW1
print user_rank
if user_rank % 10000 == 0:
ts = time.time()
print "%s : %s" % (user_rank, ts - tb)
tb = ts
def main(es):
"""
update all user in a day
"""
index_name = "20130903"
#index_name = time.strftime("%Y%m%d", time.localtime(time.time()-86400))
bool = es.indices.exists(index=index_name)
print bool
if not bool:
print "no index exist"
sys.exit(0)
user_rank = 0
bulk_action = []
n_range = range(0,100000,10000)
tb = time.time()
for left_range in n_range:
result = search_rank(index_name, left_range, 10000)
for item in result:
update_info = {}
user_rank += 1
update_info["user"] = item["_id"]
update_info["rank"] = user_rank
x = update_index_action(update_info, "rank", user_rank)
bulk_action.extend((x[0], x[1]))
if user_rank % 1000 == 0:
while 1:
try:
es.bulk(bulk_action, index=index_name, doc_type="bci", timeout=30)
bulk_action = []
break
except Exception, r:
es = ES_CLUSTER_FLOW1
print user_rank
if user_rank % 10000 == 0:
ts = time.time()
print "%s : %s" %(user_rank, ts - tb)
tb = ts
update_info["rank"] = user_rank
x = update_index_action(update_info, "rank", user_rank)
bulk_action.extend((x[0], x[1]))
if user_rank % 1000 == 0:
while 1:
try:
es.bulk(bulk_action,
index=index_name,
doc_type="bci",
timeout=30)
bulk_action = []
break
except Exception, r:
es = ES_CLUSTER_FLOW1
print user_rank
if user_rank % 10000 == 0:
ts = time.time()
print "%s : %s" % (user_rank, ts - tb)
tb = ts
if bulk_action:
es.bulk(bulk_action, index=index_name, doc_type="bci", timeout=30)
print "finish !"
if __name__ == "__main__":
main(es)
for item in result:
update_info = {}
user_rank += 1
update_info["user"] = item["_id"]
update_info["rank"] = user_rank
x = update_index_action(update_info, "rank", user_rank)
bulk_action.extend((x[0], x[1]))
if user_rank % 1000 == 0:
while 1:
try:
es.bulk(bulk_action, index=index_name, doc_type="bci", timeout=30)
bulk_action = []
break
except Exception, r:
es = ES_CLUSTER_FLOW1
print user_rank
if user_rank % 10000 == 0:
ts = time.time()
print "%s : %s" %(user_rank, ts - tb)
tb = ts
if bulk_action:
es.bulk(bulk_action, index=index_name, doc_type="bci", timeout=30)
print "finish !"
if __name__ == "__main__":
main(es)
