def stop_working_job(request):
"""停止正在进行中的转换的工作
"""
# 获取url操作
worker_id = request.matchdict['id']
thread = request.matchdict['thread']
thread_pid = request.matchdict['pid']
# pid为-1则不能杀
if thread_pid == -1:
jobs = ztq_core.get_job_state(worker_id)
task = jobs[thread]
task['runtime']['reason'] = "manual stopped"
task['runtime']['end'] = int(time.time())
ztq_core.push_runtime_error(task['runtime']['queue'], task)
del jobs[thread]
return HTTPFound(location='/workerstatus')
kill_command = {
'command': 'kill',
'timestamp': int(time.time()),
'pid': thread_pid
}
cmd_queue = ztq_core.get_command_queue(worker_id)
# 避免同时发送多条结束命令
if cmd_queue:
for command in cmd_queue:
if command.get('pid', None) == kill_command['pid']:
return HTTPFound(location='/workerstatus')
cmd_queue.push(kill_command)
return HTTPFound(location='/workerstatus')
def run(self):
self.init()
# 监听指令
commands = ztq_core.get_command_queue(self.worker_name)
while True:
try:
command = commands.pop()
if command['command'] == 'report':
worker_state = ztq_core.get_worker_state()
worker_state[self.worker_name] = report(self.login_time)
elif command['command'] == 'updatedriver':
# TODO
#async_drive_config()
pass
elif command['command'] == 'updateworker' and \
CONFIG['server'].get('active_config', 'false').lower() == 'true':
queue = ztq_core.get_worker_config()
set_job_threads(queue[self.worker_name])
elif command['command'] == 'kill':
kill_transform(pid=command['pid'], timestamp=command['timestamp'])
elif command['command'] == 'cancel':
cancel_transform(pid=command['pid'], timestamp=command['timestamp'])
except ztq_core.ConnectionError, e:
logger.error('ERROR: redis command connection error: %s' % str(e))
time.sleep(3)
except ztq_core.ResponseError, e:
logger.error('ERROR: redis command response error: %s' % str(e))
time.sleep(3)
def run(self):
self.init()
# 监听指令
commands = ztq_core.get_command_queue(self.worker_name)
while True:
try:
command = commands.pop()
if command['command'] == 'report':
worker_state = ztq_core.get_worker_state()
worker_state[self.worker_name] = report(self.login_time)
elif command['command'] == 'updatedriver':
# TODO
#async_drive_config()
pass
elif command['command'] == 'updateworker':
queue = ztq_core.get_worker_config()
set_job_threads(queue[self.worker_name])
elif command['command'] == 'kill':
kill_transform(pid=command['pid'], timestamp=command['timestamp'])
elif command['command'] == 'cancel':
cancel_transform(pid=command['pid'], timestamp=command['timestamp'])
except KeyboardInterrupt:
import os
# 实际上调用的是command_execute.clear_thread
os.sys.exitfunc()
os._exit(0)
def stop_working_job(request):
"""停止正在进行中的转换的工作
"""
# 获取url操作
worker_id = request.matchdict['id']
thread = request.matchdict['thread']
thread_pid = request.matchdict['pid']
# pid为-1则不能杀
if thread_pid == '-1':
jobs = ztq_core.get_job_state(worker_id)
task = jobs[thread]
task['runtime']['reason'] = "manual stopped"
task['runtime']['end'] = int( time.time() )
ztq_core.push_runtime_error(task['runtime']['queue'], task)
del jobs[thread]
return HTTPFound(location = '/workerstatus')
kill_command = {
'command':'kill',
'timestamp':int(time.time()),
'pid': thread_pid
}
cmd_queue = ztq_core.get_command_queue(worker_id)
# 避免同时发送多条结束命令
if cmd_queue:
for command in cmd_queue:
if command.get('pid', None) == kill_command['pid']:
return HTTPFound(location = '/workerstatus')
cmd_queue.push(kill_command)
return HTTPFound(location = '/workerstatus')
def run(self):
self.init()
# 监听指令
commands = ztq_core.get_command_queue(self.worker_name)
while True:
try:
command = commands.pop()
if command['command'] == 'report':
worker_state = ztq_core.get_worker_state()
worker_state[self.worker_name] = report(self.login_time)
elif command['command'] == 'updatedriver':
# TODO
#async_drive_config()
pass
elif command['command'] == 'updateworker':
queue = ztq_core.get_worker_config()
set_job_threads(queue[self.worker_name])
elif command['command'] == 'kill':
kill_transform(pid=command['pid'],
timestamp=command['timestamp'])
elif command['command'] == 'cancel':
cancel_transform(pid=command['pid'],
timestamp=command['timestamp'])
except KeyboardInterrupt:
import os
# 实际上调用的是command_execute.clear_thread
os.sys.exitfunc()
os._exit(0)
def send_sync_command(worker_name):
"""向转换器下达同步指令
"""
sync_command= {'command':'updateworker','timestamp':int(time.time())}
cmd_queue = ztq_core.get_command_queue(worker_name)
# 避免同时发送多条同步命令
if cmd_queue:
for command in cmd_queue:
if command.get('command', None) == sync_command['command']:
return 0
cmd_queue.push(sync_command)
def send_sync_command(worker_name):
"""向转换器下达同步指令
"""
sync_command = {'command': 'updateworker', 'timestamp': int(time.time())}
cmd_queue = ztq_core.get_command_queue(worker_name)
# 避免同时发送多条同步命令
if cmd_queue:
for command in cmd_queue:
if command.get('command', None) == sync_command['command']:
return 0
cmd_queue.push(sync_command)
def send_command(worker_name, command_stm):
"""向worker发报告状态指令
"""
send_command = {'command': command_stm, 'timestamp': int(time.time())}
cmd_queue = ztq_core.get_command_queue(worker_name)
# 避免同时发送多条同步命令
if cmd_queue:
for command in cmd_queue:
if command.get('command', None) == send_command['command']:
return 0
cmd_queue.push(send_command)
def send_command(worker_name, command_stm):
"""向worker发报告状态指令
"""
send_command= {
'command':command_stm,
'timestamp':int(time.time())
}
cmd_queue = ztq_core.get_command_queue(worker_name)
# 避免同时发送多条同步命令
if cmd_queue:
for command in cmd_queue:
if command.get('command', None) == send_command['command']:
return 0
cmd_queue.push(send_command)
def get_worker_list():
dispatcher_config = ztq_core.get_dispatcher_config()
worker_weight = dispatcher_config['worker_weight']
workers_dict = ztq_core.get_worker_state().items()
for worker_name, worker_status in workers_dict:
worker_status['_worker_name'] = worker_name
worker_status['_started'] = \
datetime.datetime.fromtimestamp(worker_status['started'])
worker_status['_timestamp'] = \
datetime.datetime.fromtimestamp(worker_status['timestamp'])
worker_status['_worker_weight'] = worker_weight.get(worker_name, 0)
# 检查worker是否在工作
cmd_queue = ztq_core.get_command_queue(worker_name)
# 如果指令队列不为空的话,意味着worker没工作,属于下线状态
if cmd_queue: worker_status['_active'] = u'shutdown'
elif worker_status['_worker_weight'] == 0:
worker_status['_active'] = u'ldle'
else:
worker_status['_active'] = u'work'
# 获取worker开了多少个线程
worker_job = ztq_core.get_job_state(worker_name)
worker_status['_threads'] = []
for thread_name, thread_status in worker_job.items():
thread_status['_detail'] = pprint.pformat(thread_status)
thread_status['_name'] = thread_name
thread_status['_comment'] = thread_status['kw'].get(
'comment', thread_status['process'].get('comment', ''))
thread_status['_pid'] = thread_status['process'].get('pid', -1)
ident = unicode(thread_status['process'].get('ident', -1))
if ident in worker_status['traceback']:
thread_status['_thread_detail'] = pprint.pformat(
worker_status['traceback'][ident])
# 任务进行了多少时间
used_time = int(time.time()) - thread_status['process']['start']
if used_time > 3600:
used_time = u'%.2f小时' % (used_time / 3600.0)
elif used_time > 60:
used_time = u'%.2f分钟' % (used_time / 60.0)
thread_status['_take_time'] = used_time
worker_status['_threads'].append(thread_status)
yield worker_status
def get_worker_list():
workers_dict = ztq_core.get_worker_state().items()
for worker_name, worker_status in workers_dict:
worker_status['_worker_name'] = worker_name
worker_status['_started'] = \
datetime.datetime.fromtimestamp(worker_status['started'])
worker_status['_timestamp'] = \
datetime.datetime.fromtimestamp(worker_status['timestamp'])
# 检查worker是否在工作
cmd_queue = ztq_core.get_command_queue(worker_name)
# 如果指令队列不为空的话,意味着worker没工作,属于下线状态
if cmd_queue:
worker_status['_active'] = u'shutdown'
else:
worker_status['_active'] = u'work'
# 获取worker开了多少个线程
worker_job = ztq_core.get_job_state(worker_name)
worker_status['_threads'] = []
for thread_name,thread_status in worker_job.items():
thread_status['_detail'] = pprint.pformat(thread_status)
thread_status['_name'] = thread_name
thread_status['_comment'] = thread_status['kw'].get('comment',thread_status['process'].get('comment', ''))
thread_status['_pid'] = thread_status['process'].get('pid', -1)
ident = unicode(thread_status['process'].get('ident', -1))
if ident in worker_status['traceback']:
thread_status['_thread_detail'] = pprint.pformat(worker_status['traceback'][ident])
# 任务进行了多少时间
used_time = int(time.time())-thread_status['process']['start']
if used_time > 3600:
used_time = u'%.2f小时' % (used_time / 3600.0)
elif used_time > 60:
used_time = u'%.2f分钟' % (used_time / 60.0)
thread_status['_take_time'] = used_time
worker_status['_threads'].append(thread_status)
yield worker_status
def stop_working_job(request):
"""停止正在进行中的转换的工作
"""
kill_command = {
'command': 'kill',
'timestamp': int(time.time()),
'pid': '',
}
# 获取url操作
worker_id = request.matchdict['id']
thread_pid = request.matchdict['pid']
# pid为-1则不能杀
if thread_pid == -1: return HTTPFound(location='/workerstatus')
else: kill_command['pid'] = thread_pid
cmd_queue = ztq_core.get_command_queue(worker_id)
# 避免同时发送多条结束命令
if cmd_queue:
for command in cmd_queue:
if command.get('pid', None) == kill_command['pid']:
return HTTPFound(location='/workerstatus')
cmd_queue.push(kill_command)
return HTTPFound(location='/workerstatus')
def stop_working_job(request):
"""停止正在进行中的转换的工作
"""
kill_command = {
'command':'kill',
'timestamp':int(time.time()),
'pid':'',
}
# 获取url操作
worker_id = request.matchdict['id']
thread_pid = request.matchdict['pid']
# pid为-1则不能杀
if thread_pid == -1: return HTTPFound(location = '/workerstatus')
else: kill_command['pid'] = thread_pid
cmd_queue = ztq_core.get_command_queue(worker_id)
# 避免同时发送多条结束命令
if cmd_queue:
for command in cmd_queue:
if command.get('pid', None) == kill_command['pid']:
return HTTPFound(location = '/workerstatus')
cmd_queue.push(kill_command)
return HTTPFound(location = '/workerstatus')
