def stop_working_job(request): """停止正在进行中的转换的工作 """ # 获取url操作 worker_id = request.matchdict['id'] thread = request.matchdict['thread'] thread_pid = request.matchdict['pid'] # pid为-1则不能杀 if thread_pid == -1: jobs = ztq_core.get_job_state(worker_id) task = jobs[thread] task['runtime']['reason'] = "manual stopped" task['runtime']['end'] = int(time.time()) ztq_core.push_runtime_error(task['runtime']['queue'], task) del jobs[thread] return HTTPFound(location='/workerstatus') kill_command = { 'command': 'kill', 'timestamp': int(time.time()), 'pid': thread_pid } cmd_queue = ztq_core.get_command_queue(worker_id) # 避免同时发送多条结束命令 if cmd_queue: for command in cmd_queue: if command.get('pid', None) == kill_command['pid']: return HTTPFound(location='/workerstatus') cmd_queue.push(kill_command) return HTTPFound(location='/workerstatus')
def run(self): self.init() # 监听指令 commands = ztq_core.get_command_queue(self.worker_name) while True: try: command = commands.pop() if command['command'] == 'report': worker_state = ztq_core.get_worker_state() worker_state[self.worker_name] = report(self.login_time) elif command['command'] == 'updatedriver': # TODO #async_drive_config() pass elif command['command'] == 'updateworker' and \ CONFIG['server'].get('active_config', 'false').lower() == 'true': queue = ztq_core.get_worker_config() set_job_threads(queue[self.worker_name]) elif command['command'] == 'kill': kill_transform(pid=command['pid'], timestamp=command['timestamp']) elif command['command'] == 'cancel': cancel_transform(pid=command['pid'], timestamp=command['timestamp']) except ztq_core.ConnectionError, e: logger.error('ERROR: redis command connection error: %s' % str(e)) time.sleep(3) except ztq_core.ResponseError, e: logger.error('ERROR: redis command response error: %s' % str(e)) time.sleep(3)
def run(self): self.init() # 监听指令 commands = ztq_core.get_command_queue(self.worker_name) while True: try: command = commands.pop() if command['command'] == 'report': worker_state = ztq_core.get_worker_state() worker_state[self.worker_name] = report(self.login_time) elif command['command'] == 'updatedriver': # TODO #async_drive_config() pass elif command['command'] == 'updateworker': queue = ztq_core.get_worker_config() set_job_threads(queue[self.worker_name]) elif command['command'] == 'kill': kill_transform(pid=command['pid'], timestamp=command['timestamp']) elif command['command'] == 'cancel': cancel_transform(pid=command['pid'], timestamp=command['timestamp']) except KeyboardInterrupt: import os # 实际上调用的是command_execute.clear_thread os.sys.exitfunc() os._exit(0)
def stop_working_job(request): """停止正在进行中的转换的工作 """ # 获取url操作 worker_id = request.matchdict['id'] thread = request.matchdict['thread'] thread_pid = request.matchdict['pid'] # pid为-1则不能杀 if thread_pid == '-1': jobs = ztq_core.get_job_state(worker_id) task = jobs[thread] task['runtime']['reason'] = "manual stopped" task['runtime']['end'] = int( time.time() ) ztq_core.push_runtime_error(task['runtime']['queue'], task) del jobs[thread] return HTTPFound(location = '/workerstatus') kill_command = { 'command':'kill', 'timestamp':int(time.time()), 'pid': thread_pid } cmd_queue = ztq_core.get_command_queue(worker_id) # 避免同时发送多条结束命令 if cmd_queue: for command in cmd_queue: if command.get('pid', None) == kill_command['pid']: return HTTPFound(location = '/workerstatus') cmd_queue.push(kill_command) return HTTPFound(location = '/workerstatus')
def send_sync_command(worker_name): """向转换器下达同步指令 """ sync_command= {'command':'updateworker','timestamp':int(time.time())} cmd_queue = ztq_core.get_command_queue(worker_name) # 避免同时发送多条同步命令 if cmd_queue: for command in cmd_queue: if command.get('command', None) == sync_command['command']: return 0 cmd_queue.push(sync_command)
def send_sync_command(worker_name): """向转换器下达同步指令 """ sync_command = {'command': 'updateworker', 'timestamp': int(time.time())} cmd_queue = ztq_core.get_command_queue(worker_name) # 避免同时发送多条同步命令 if cmd_queue: for command in cmd_queue: if command.get('command', None) == sync_command['command']: return 0 cmd_queue.push(sync_command)
def send_command(worker_name, command_stm): """向worker发报告状态指令 """ send_command = {'command': command_stm, 'timestamp': int(time.time())} cmd_queue = ztq_core.get_command_queue(worker_name) # 避免同时发送多条同步命令 if cmd_queue: for command in cmd_queue: if command.get('command', None) == send_command['command']: return 0 cmd_queue.push(send_command)
def send_command(worker_name, command_stm): """向worker发报告状态指令 """ send_command= { 'command':command_stm, 'timestamp':int(time.time()) } cmd_queue = ztq_core.get_command_queue(worker_name) # 避免同时发送多条同步命令 if cmd_queue: for command in cmd_queue: if command.get('command', None) == send_command['command']: return 0 cmd_queue.push(send_command)
def get_worker_list(): dispatcher_config = ztq_core.get_dispatcher_config() worker_weight = dispatcher_config['worker_weight'] workers_dict = ztq_core.get_worker_state().items() for worker_name, worker_status in workers_dict: worker_status['_worker_name'] = worker_name worker_status['_started'] = \ datetime.datetime.fromtimestamp(worker_status['started']) worker_status['_timestamp'] = \ datetime.datetime.fromtimestamp(worker_status['timestamp']) worker_status['_worker_weight'] = worker_weight.get(worker_name, 0) # 检查worker是否在工作 cmd_queue = ztq_core.get_command_queue(worker_name) # 如果指令队列不为空的话,意味着worker没工作,属于下线状态 if cmd_queue: worker_status['_active'] = u'shutdown' elif worker_status['_worker_weight'] == 0: worker_status['_active'] = u'ldle' else: worker_status['_active'] = u'work' # 获取worker开了多少个线程 worker_job = ztq_core.get_job_state(worker_name) worker_status['_threads'] = [] for thread_name, thread_status in worker_job.items(): thread_status['_detail'] = pprint.pformat(thread_status) thread_status['_name'] = thread_name thread_status['_comment'] = thread_status['kw'].get( 'comment', thread_status['process'].get('comment', '')) thread_status['_pid'] = thread_status['process'].get('pid', -1) ident = unicode(thread_status['process'].get('ident', -1)) if ident in worker_status['traceback']: thread_status['_thread_detail'] = pprint.pformat( worker_status['traceback'][ident]) # 任务进行了多少时间 used_time = int(time.time()) - thread_status['process']['start'] if used_time > 3600: used_time = u'%.2f小时' % (used_time / 3600.0) elif used_time > 60: used_time = u'%.2f分钟' % (used_time / 60.0) thread_status['_take_time'] = used_time worker_status['_threads'].append(thread_status) yield worker_status
def get_worker_list(): workers_dict = ztq_core.get_worker_state().items() for worker_name, worker_status in workers_dict: worker_status['_worker_name'] = worker_name worker_status['_started'] = \ datetime.datetime.fromtimestamp(worker_status['started']) worker_status['_timestamp'] = \ datetime.datetime.fromtimestamp(worker_status['timestamp']) # 检查worker是否在工作 cmd_queue = ztq_core.get_command_queue(worker_name) # 如果指令队列不为空的话,意味着worker没工作,属于下线状态 if cmd_queue: worker_status['_active'] = u'shutdown' else: worker_status['_active'] = u'work' # 获取worker开了多少个线程 worker_job = ztq_core.get_job_state(worker_name) worker_status['_threads'] = [] for thread_name,thread_status in worker_job.items(): thread_status['_detail'] = pprint.pformat(thread_status) thread_status['_name'] = thread_name thread_status['_comment'] = thread_status['kw'].get('comment',thread_status['process'].get('comment', '')) thread_status['_pid'] = thread_status['process'].get('pid', -1) ident = unicode(thread_status['process'].get('ident', -1)) if ident in worker_status['traceback']: thread_status['_thread_detail'] = pprint.pformat(worker_status['traceback'][ident]) # 任务进行了多少时间 used_time = int(time.time())-thread_status['process']['start'] if used_time > 3600: used_time = u'%.2f小时' % (used_time / 3600.0) elif used_time > 60: used_time = u'%.2f分钟' % (used_time / 60.0) thread_status['_take_time'] = used_time worker_status['_threads'].append(thread_status) yield worker_status
def stop_working_job(request): """停止正在进行中的转换的工作 """ kill_command = { 'command': 'kill', 'timestamp': int(time.time()), 'pid': '', } # 获取url操作 worker_id = request.matchdict['id'] thread_pid = request.matchdict['pid'] # pid为-1则不能杀 if thread_pid == -1: return HTTPFound(location='/workerstatus') else: kill_command['pid'] = thread_pid cmd_queue = ztq_core.get_command_queue(worker_id) # 避免同时发送多条结束命令 if cmd_queue: for command in cmd_queue: if command.get('pid', None) == kill_command['pid']: return HTTPFound(location='/workerstatus') cmd_queue.push(kill_command) return HTTPFound(location='/workerstatus')
def stop_working_job(request): """停止正在进行中的转换的工作 """ kill_command = { 'command':'kill', 'timestamp':int(time.time()), 'pid':'', } # 获取url操作 worker_id = request.matchdict['id'] thread_pid = request.matchdict['pid'] # pid为-1则不能杀 if thread_pid == -1: return HTTPFound(location = '/workerstatus') else: kill_command['pid'] = thread_pid cmd_queue = ztq_core.get_command_queue(worker_id) # 避免同时发送多条结束命令 if cmd_queue: for command in cmd_queue: if command.get('pid', None) == kill_command['pid']: return HTTPFound(location = '/workerstatus') cmd_queue.push(kill_command) return HTTPFound(location = '/workerstatus')