def task_jobs_handler(request):
"""将任务调整到队头或者队尾
传入参数:http://server/taskqueues/q01/job?action=high_priority&hash_id={{job_hash_id}}
"""
valid_action = ('high_priority', 'low_priority', 'delete')
queue_name = request.matchdict['id']
url_action = request.params.get('action', '')
job_hash_id = urllib.unquote(request.params.get('hash_id').encode('utf8'))
if url_action in valid_action:
if url_action == 'high_priority':
job_queue = ztq_core.get_task_queue(queue_name)
job_queue.remove(job_hash_id)
job_queue.push(job_hash_id, to_left=False)
elif url_action == 'low_priority':
job_queue = ztq_core.get_task_queue(queue_name)
job_queue.remove(job_hash_id)
job_queue.push(job_hash_id)
elif url_action == 'delete':
job_queue = ztq_core.get_task_queue(queue_name)
job_queue.remove(job_hash_id)
job_hash = ztq_core.get_task_hash(queue_name)
job_hash.pop(job_hash_id)
return HTTPFound(location='/taskqueues/' + queue_name)
else:
return Response('Invalid request')
def task_jobs_handler(request):
"""将任务调整到队头或者队尾
传入参数:http://server/taskqueues/q01/job?action=high_priority&hash_id={{job_hash_id}}
"""
valid_action = ('high_priority','low_priority', 'delete')
queue_name = request.matchdict['id']
url_action = request.params.get('action','')
job_hash_id = urllib.unquote(request.params.get('hash_id').encode('utf8'))
if url_action in valid_action:
if url_action == 'high_priority':
job_queue = ztq_core.get_task_queue(queue_name)
job_queue.remove(job_hash_id)
job_queue.push(job_hash_id, to_left=False)
elif url_action == 'low_priority':
job_queue = ztq_core.get_task_queue(queue_name)
job_queue.remove(job_hash_id)
job_queue.push(job_hash_id)
elif url_action == 'delete':
job_queue = ztq_core.get_task_queue(queue_name)
job_queue.remove(job_hash_id)
job_hash = ztq_core.get_task_hash(queue_name)
job_hash.pop(job_hash_id)
return HTTPFound(location = '/taskqueues/'+queue_name)
else:
return Response('Invalid request')
def clear_transform_thread(threads=None):
""" clear job_threads and buffer_thread """
threads = threads or job_thread_manager.threads
names = threads.keys()
job_threads = threads.values()
# 退出buffer 线程
if buffer_thread_instance is not None:
buffer_thread_instance.stop()
sys.stdout.write('wait the buffer thread stop...\n')
# 将进程的stop 标志 设置为True
map(job_thread_manager.stop, names)
# 如果这个线程没有工作，只是在阻塞等待任务，就发送一个空的任务
# 让这个线程立刻结束
for job_thread in job_threads:
if job_thread.start_job_time == 0:
queue_name = job_thread.queue_name
queue = ztq_core.get_task_queue(queue_name)
queue.push('')
# 等待线程退出
for job_thread in job_threads:
sys.stdout.write('wait the %s stop...\n'%job_thread.getName())
job_thread.join(30)
def clear_transform_thread(threads=None):
""" clear job_threads and buffer_thread """
threads = threads or job_thread_manager.threads
names = threads.keys()
job_threads = threads.values()
# 退出buffer 线程
if buffer_thread_instance is not None:
buffer_thread_instance.stop()
sys.stdout.write('wait the buffer thread stop...\n')
# 将进程的stop 标志 设置为True
map(job_thread_manager.stop, names)
# 如果这个线程没有工作，只是在阻塞等待任务，就发送一个空的任务
# 让这个线程立刻结束
for job_thread in job_threads:
if job_thread.start_job_time == 0:
queue_name = job_thread.queue_name
queue = ztq_core.get_task_queue(queue_name)
queue.push('')
# 等待线程退出
for job_thread in job_threads:
sys.stdout.write('wait the %s stop...\n' % job_thread.getName())
job_thread.join(30)
def run(self):
if not self.config:
return
while not self._stop:
for buffer_name, buffer_config in self.config.items():
# 需要停止
if self._stop:
return
self.buffer_queue = ztq_core.get_buffer_queue(buffer_name)
self.task_queue = ztq_core.get_task_queue(buffer_name)
self.buffer_name = buffer_name
self.task_queue_limit = int(buffer_config["thread_limit"])
while True:
try:
self.start_job()
break
except ztq_core.ConnectionError:
time.sleep(3)
time.sleep(1)
def testJsonList(self):
"""Test queue connect
"""
self.queue = ztq_core.get_task_queue('q01')
self.queue.append(self.testmessage)
revmessage = self.queue.pop()
self.assertEqual(revmessage, self.testmessage)
def testJsonList(self):
"""Test queue connect
"""
self.queue = ztq_core.get_task_queue('q01')
self.queue.append(self.testmessage)
revmessage = self.queue.pop()
self.assertEqual(revmessage,self.testmessage)
def get_queues_jobs(queue_name):
queue = ztq_core.get_task_queue(queue_name)
for task_job_hash in queue.reverse():
task_job = ztq_core.get_task_hash(queue_name).get(task_job_hash)
tmp_job={}
tmp_job['_queue_name'] = queue_name
tmp_job['_id'] = urllib.quote(task_job_hash)
#tmp_job['_ori'] = task_job
tmp_job['_detail'] = pprint.pformat(task_job)
tmp_job['_created'] = datetime.datetime.fromtimestamp(task_job['runtime'].get('create', 0))
yield tmp_job
def get_queues_jobs(queue_name):
queue = ztq_core.get_task_queue(queue_name)
for task_job_hash in queue.reverse():
task_job = ztq_core.get_task_hash(queue_name).get(task_job_hash)
tmp_job = {}
tmp_job['_queue_name'] = queue_name
tmp_job['_id'] = urllib.quote(task_job_hash)
#tmp_job['_ori'] = task_job
tmp_job['_detail'] = pprint.pformat(task_job)
tmp_job['_created'] = datetime.datetime.fromtimestamp(
task_job['runtime'].get('create', 0))
yield tmp_job
def get_taskqueues_list():
# 队列情况列表
dispatcher_config = ztq_core.get_dispatcher_config()
queue_weight = dispatcher_config['queue_weight']
queues_list = ztq_core.get_queue_config()
# 排序
sort_queue_name = {}
for queue_name, queue_config in queues_list.items():
sort_queue_name[queue_name] = len(ztq_core.get_error_queue(queue_name))
for queue_name in sorted(sort_queue_name,
key=lambda x: sort_queue_name[x],
reverse=True):
task_queue = {}
task_queue['name'] = queue_name
#task_queue['tags'] = queue_config.get('tags',())
queue = ztq_core.get_task_queue(queue_name)
# 任务数/错误数
task_queue['length'] = len(queue)
task_queue['error_length'] = sort_queue_name[queue_name]
#任务首个时间
task_queue['error_end'] = task_queue['first'] = ''
first_job = queue[0]
first_job = ztq_core.get_task_hash(queue_name).get(first_job)
if first_job:
task_queue['first'] = datetime.datetime.fromtimestamp(
first_job['runtime'].get('create', 0))
#错误最末一个的时间
error_first_job = ztq_core.get_error_queue(queue_name)[0]
error_first_job = ztq_core.get_error_hash(queue_name).get(
error_first_job)
if error_first_job:
task_queue['error_end'] = datetime.datetime.fromtimestamp(
error_first_job['runtime'].get('create', 0))
task_queue['weight'] = queue_weight.get(queue_name, 0)
# 获取worker工作线程配置
workers_config = ztq_core.get_worker_config()
task_queue['from_right'] = True
for worker_name, worker_config in workers_config.items():
task_queue['workers'] = []
for config in worker_config.get(queue_name, []):
task_queue['workers'].append(
[worker_name + ':', config['interval']])
if 'from_right' in config:
task_queue['from_right'] = config['from_right']
task_queue['buffer_length'] = len(
ztq_core.get_buffer_queue(queue_name))
yield task_queue
def task_queues(request):
"""查看转换队列运行状态
传出参数:所有原子队列的运行转换
"""
task_job_length = 0
error_job_length = 0
# 计算原子队列,原始队列和错误队列的总长度
queues_list = ztq_core.get_queue_config()
for queue_name, queue_config in queues_list.items():
task_job_length += len(ztq_core.get_task_queue(queue_name))
error_job_length += len(ztq_core.get_error_queue(queue_name))
task_queues = utils.get_taskqueues_list()
return {'task_queues':task_queues,
'task_job_length':task_job_length,
'error_job_length':error_job_length, }
def get_taskqueues_list():
# 队列情况列表
dispatcher_config = ztq_core.get_dispatcher_config()
queue_weight = dispatcher_config['queue_weight']
queues_list = ztq_core.get_queue_config()
# 排序
sort_queue_name = {}
for queue_name, queue_config in queues_list.items():
sort_queue_name[queue_name] = len(ztq_core.get_error_queue(queue_name))
for queue_name in sorted(sort_queue_name,
key=lambda x: sort_queue_name[x],
reverse=True):
task_queue = {}
task_queue['name'] = queue_name
#task_queue['tags'] = queue_config.get('tags',())
queue = ztq_core.get_task_queue(queue_name)
# 任务数/错误数
task_queue['length'] = len(queue)
task_queue['error_length'] = sort_queue_name[queue_name]
#任务首个时间
task_queue['error_end'] = task_queue['first'] = ''
first_job = queue[0]
first_job= ztq_core.get_task_hash(queue_name).get(first_job)
if first_job:
task_queue['first'] = datetime.datetime.fromtimestamp(first_job['runtime'].get('create', 0))
#错误最末一个的时间
error_first_job = ztq_core.get_error_queue(queue_name)[0]
error_first_job = ztq_core.get_error_hash(queue_name).get(error_first_job)
if error_first_job:
task_queue['error_end'] = datetime.datetime.fromtimestamp(error_first_job['runtime'].get('create', 0))
task_queue['weight'] = queue_weight.get(queue_name, 0)
# 获取worker工作线程配置
workers_config = ztq_core.get_worker_config()
task_queue['from_right'] = True
for worker_name,worker_config in workers_config.items():
task_queue['workers'] = []
for config in worker_config.get(queue_name,[]):
task_queue['workers'].append([worker_name+':', config['interval']])
if 'from_right' in config:
task_queue['from_right'] = config['from_right']
task_queue['buffer_length'] = len(ztq_core.get_buffer_queue(queue_name))
yield task_queue
def task_queues(request):
"""查看转换队列运行状态
传出参数:所有原子队列的运行转换
"""
task_job_length = 0
error_job_length = 0
# 计算原子队列,原始队列和错误队列的总长度
queues_list = ztq_core.get_queue_config()
for queue_name, queue_config in queues_list.items():
task_job_length += len(ztq_core.get_task_queue(queue_name))
error_job_length += len(ztq_core.get_error_queue(queue_name))
task_queues = utils.get_taskqueues_list()
return {
'task_queues': task_queues,
'task_job_length': task_job_length,
'error_job_length': error_job_length,
}
def run(self):
if not self.config: return
while not self._stop:
for buffer_name, buffer_config in self.config.items():
# 需要停止
if self._stop: return
self.buffer_queue = ztq_core.get_buffer_queue(buffer_name)
self.task_queue = ztq_core.get_task_queue(buffer_name)
self.buffer_name = buffer_name
self.task_queue_limit = int(buffer_config['thread_limit'])
while True:
try:
self.start_job()
break
except ztq_core.ConnectionError:
time.sleep(3)
time.sleep(1)