def main(es): """ update all user in a day """ index_name = "20130903" #index_name = time.strftime("%Y%m%d", time.localtime(time.time()-86400)) bool = es.indices.exists(index=index_name) print bool if not bool: print "no index exist" sys.exit(0) user_rank = 0 bulk_action = [] n_range = range(0, 100000, 10000) tb = time.time() for left_range in n_range: result = search_rank(index_name, left_range, 10000) for item in result: update_info = {} user_rank += 1 update_info["user"] = item["_id"] update_info["rank"] = user_rank x = update_index_action(update_info, "rank", user_rank) bulk_action.extend((x[0], x[1])) if user_rank % 1000 == 0: while 1: try: es.bulk(bulk_action, index=index_name, doc_type="bci", timeout=30) bulk_action = [] break except Exception, r: es = ES_CLUSTER_FLOW1 print user_rank if user_rank % 10000 == 0: ts = time.time() print "%s : %s" % (user_rank, ts - tb) tb = ts
def main(es): """ update all user in a day """ index_name = "20130903" #index_name = time.strftime("%Y%m%d", time.localtime(time.time()-86400)) bool = es.indices.exists(index=index_name) print bool if not bool: print "no index exist" sys.exit(0) user_rank = 0 bulk_action = [] n_range = range(0,100000,10000) tb = time.time() for left_range in n_range: result = search_rank(index_name, left_range, 10000) for item in result: update_info = {} user_rank += 1 update_info["user"] = item["_id"] update_info["rank"] = user_rank x = update_index_action(update_info, "rank", user_rank) bulk_action.extend((x[0], x[1])) if user_rank % 1000 == 0: while 1: try: es.bulk(bulk_action, index=index_name, doc_type="bci", timeout=30) bulk_action = [] break except Exception, r: es = ES_CLUSTER_FLOW1 print user_rank if user_rank % 10000 == 0: ts = time.time() print "%s : %s" %(user_rank, ts - tb) tb = ts
update_info["rank"] = user_rank x = update_index_action(update_info, "rank", user_rank) bulk_action.extend((x[0], x[1])) if user_rank % 1000 == 0: while 1: try: es.bulk(bulk_action, index=index_name, doc_type="bci", timeout=30) bulk_action = [] break except Exception, r: es = ES_CLUSTER_FLOW1 print user_rank if user_rank % 10000 == 0: ts = time.time() print "%s : %s" % (user_rank, ts - tb) tb = ts if bulk_action: es.bulk(bulk_action, index=index_name, doc_type="bci", timeout=30) print "finish !" if __name__ == "__main__": main(es)
for item in result: update_info = {} user_rank += 1 update_info["user"] = item["_id"] update_info["rank"] = user_rank x = update_index_action(update_info, "rank", user_rank) bulk_action.extend((x[0], x[1])) if user_rank % 1000 == 0: while 1: try: es.bulk(bulk_action, index=index_name, doc_type="bci", timeout=30) bulk_action = [] break except Exception, r: es = ES_CLUSTER_FLOW1 print user_rank if user_rank % 10000 == 0: ts = time.time() print "%s : %s" %(user_rank, ts - tb) tb = ts if bulk_action: es.bulk(bulk_action, index=index_name, doc_type="bci", timeout=30) print "finish !" if __name__ == "__main__": main(es)