def f(redis_task):
redis_task_temp = load_batch_data(redis_task)
print 'redis_task_temp: ', redis_task_temp, r.llen(redis_task_temp)
if redis_task_temp:
length = r.llen(redis_task_temp)
while r.llen(redis_task_temp):
l = r.rpop(redis_task_temp)
print l
remove_batch_data(redis_task, length)
def load_batch_data(redis_task):
redis_task_temp = redis_task + '_temp'
length = r.llen(redis_task)
temp = r.lrange(redis_task, length-batch, length-1)
if temp:
for t in temp:
r.rpush(redis_task_temp, eval(t))
return redis_task_temp
return False
index_pre = value
elif op == '-u':
user_index = value
elif op == '-r':
redis_task = value
print 'translate task details: ', index_pre, user_index, redis_task
queue = Manager().Queue()
#判断时text类型的翻译任务还是user类型的翻译任务，分而治之
if index_pre in ['twitter_flow_text_', 'facebook_flow_text_']:
bulk_action_all = {}
count_dict = {}
count_i = 0
text_index_pre = index_pre
redis_task_temp = load_batch_data(redis_task)
length = r.llen(redis_task_temp)
while r.llen(redis_task_temp):
print 'trans start'
p = Pool()
while r.llen(redis_task_temp):
li = []
for i in range(gap):
l = r.rpop(redis_task_temp)
if l:
li.append(eval(l))
if li:
p.apply_async(subprocess_task, args=(li,queue,redis_task_temp,))
p.close()
p.join()
print 'trans end'
while not queue.empty():
