def f(redis_task):
redis_task_temp = load_batch_data(redis_task)
print 'redis_task_temp: ', redis_task_temp, r.llen(redis_task_temp)
if redis_task_temp:
length = r.llen(redis_task_temp)
while r.llen(redis_task_temp):
l = r.rpop(redis_task_temp)
print l
remove_batch_data(redis_task, length)
def remove_batch_data(redis_task, length):
redis_task_temp = redis_task + '_temp'
for i in range(length):
r.rpop(redis_task)
r.delete(redis_task_temp)
#判断时text类型的翻译任务还是user类型的翻译任务，分而治之
if index_pre in ['twitter_flow_text_', 'facebook_flow_text_']:
bulk_action_all = {}
count_dict = {}
count_i = 0
text_index_pre = index_pre
redis_task_temp = load_batch_data(redis_task)
length = r.llen(redis_task_temp)
while r.llen(redis_task_temp):
print 'trans start'
p = Pool()
while r.llen(redis_task_temp):
li = []
for i in range(gap):
l = r.rpop(redis_task_temp)
if l:
li.append(eval(l))
if li:
p.apply_async(subprocess_task, args=(li,queue,redis_task_temp,))
p.close()
p.join()
print 'trans end'
while not queue.empty():
item = queue.get(True)
_id = item[0]
text_ch = item[1]
flag_ch = item[2]
date_time = item[3]
action = {'update':{'_id': _id}}
source_item = {'text_ch': text_ch, 'flag_ch': flag_ch}