class ConnectionQueue(object): """ Holds connections to resources. Each time it's called a connection is fetched from its underlying queue assuming any connection is still available. """ def __init__(self, pool_size, queue_build_cap, conn_name, conn_type, address, add_client_func): self.queue = Queue(pool_size) self.queue_build_cap = queue_build_cap self.conn_name = conn_name self.conn_type = conn_type self.address = address self.add_client_func = add_client_func self.logger = logging.getLogger(self.__class__.__name__) def __call__(self): return _Connection(self.queue, self.conn_name) def put_client(self, client): self.queue.put(client) self.logger.info('Added `%s` client to %s (%s)', self.conn_name, self.address, self.conn_type) def build_queue(self): """ Spawns greenlets to populate the queue and waits up to self.queue_build_cap seconds until the queue is full. If it never is, raises an exception stating so. """ for x in range(self.queue.maxsize): gevent.spawn(self.add_client_func) start = datetime.utcnow() build_until = start + timedelta(seconds=self.queue_build_cap) while not self.queue.full(): gevent.sleep(0.5) now = datetime.utcnow() if now >= build_until: self.logger.error( 'Built %s/%s %s clients to `%s` within %s seconds, giving up', self.queue.qsize(), self.queue.maxsize, self.conn_type, self.address, self.queue_build_cap) return self.logger.info( '%d/%d %s clients connected to `%s` (%s) after %s (cap: %ss)', self.queue.qsize(), self.queue.maxsize, self.conn_type, self.address, self.conn_name, now - start, self.queue_build_cap) self.logger.info('Obtained %d %s clients to `%s` for `%s`', self.queue.maxsize, self.conn_type, self.address, self.conn_name)
class PSPool(object): LIFE_TIMES = 60 * 1 #sock生命周期 def __init__(self, host, port, max_sock): self.host = host self.port = port self.max_sock = max_sock self.socks = Queue(maxsize=max_sock) self.threads = {} self.sock_times = {} def init_sock(self): """ 初始化和pp服务器的socket """ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((self.host, self.port)) return sock def get(self): if self.socks.empty(): sock = self.init_sock() self.sock_times[sock] = time.time() return sock return self.socks.get() def put(self, sock): times = time.time() - self.sock_times[sock] if times >= self.LIFE_TIMES or self.socks.full(): self.free(sock) return self.socks.put(sock) def free(self, sock): self.sock_times.pop(sock, None) try: sock.close() except: pass def __enter__(self): cur_thread = getcurrent() if cur_thread in self.threads: raise ValueError('not support reenter') self.threads[cur_thread] = self.get() return self.threads[cur_thread] def __exit__(self, exc_type, exc_val, exc_tb): sock = self.threads.pop(getcurrent()) if exc_type is None: self.put(sock) else: self.free(sock)
class ConnectionQueue(object): """ Holds connections to resources. Each time it's called a connection is fetched from its underlying queue assuming any connection is still available. """ def __init__(self, pool_size, queue_build_cap, conn_name, conn_type, address, add_client_func): self.queue = Queue(pool_size) self.queue_build_cap = queue_build_cap self.conn_name = conn_name self.conn_type = conn_type self.address = address self.add_client_func = add_client_func self.logger = logging.getLogger(self.__class__.__name__) def __call__(self): return _Connection(self.queue, self.conn_name) def put_client(self, client): self.queue.put(client) self.logger.info('Added `%s` client to %s (%s)', self.conn_name, self.address, self.conn_type) def build_queue(self): """ Spawns greenlets to populate the queue and waits up to self.queue_build_cap seconds until the queue is full. If it never is, raises an exception stating so. """ for x in range(self.queue.maxsize): gevent.spawn(self.add_client_func) start = datetime.utcnow() build_until = start + timedelta(seconds=self.queue_build_cap) while not self.queue.full(): gevent.sleep(0.5) now = datetime.utcnow() if now >= build_until: self.logger.error('Built %s/%s %s clients to `%s` within %s seconds, giving up', self.queue.qsize(), self.queue.maxsize, self.conn_type, self.address, self.queue_build_cap) return self.logger.info('%d/%d %s clients connected to `%s` (%s) after %s (cap: %ss)', self.queue.qsize(), self.queue.maxsize, self.conn_type, self.address, self.conn_name, now - start, self.queue_build_cap) self.logger.info('Obtained %d %s clients to `%s` for `%s`', self.queue.maxsize, self.conn_type, self.address, self.conn_name)
class ConnectionQueue(object): """ Holds connections to resources. Each time it's called a connection is fetched from its underlying queue assuming any connection is still available. """ def __init__(self, pool_size, queue_build_cap, conn_name, conn_type, address, add_client_func): self.queue = Queue(pool_size) self.queue_build_cap = queue_build_cap self.conn_name = conn_name self.conn_type = conn_type self.address = address self.add_client_func = add_client_func self.keep_connecting = True self.lock = RLock() # How many add_client_func instances are running currently, # must be updated with self.lock held. self.in_progress_count = 0 self.logger = logging.getLogger(self.__class__.__name__) def __call__(self): return _Connection(self.queue, self.conn_name) def put_client(self, client): with self.lock: if self.queue.full(): is_accepted = False msg = 'Skipped adding a superfluous `%s` client to %s (%s)' log_func = self.logger.info else: self.queue.put(client) is_accepted = True msg = 'Added `%s` client to %s (%s)' log_func = self.logger.info log_func(msg, self.conn_name, self.address, self.conn_type) return is_accepted def _build_queue(self): start = datetime.utcnow() build_until = start + timedelta(seconds=self.queue_build_cap) suffix = 's ' if self.queue.maxsize > 1 else ' ' try: while self.keep_connecting and not self.queue.full(): gevent.sleep(5) now = datetime.utcnow() self.logger.info( '%d/%d %s clients obtained to `%s` (%s) after %s (cap: %ss)', self.queue.qsize(), self.queue.maxsize, self.conn_type, self.address, self.conn_name, now - start, self.queue_build_cap) if now >= build_until: # Log the fact that the queue is not full yet self.logger.info( 'Built %s/%s %s clients to `%s` within %s seconds, sleeping until %s (UTC)', self.queue.qsize(), self.queue.maxsize, self.conn_type, self.address, self.queue_build_cap, datetime.utcnow() + timedelta(seconds=self.queue_build_cap)) # Sleep for a predetermined time gevent.sleep(self.queue_build_cap) # Spawn additional greenlets to fill up the queue but make sure not to spawn # more greenlets than there are slots in the queue still available. with self.lock: if self.in_progress_count < self.queue.maxsize: self._spawn_add_client_func(self.queue.maxsize - self.in_progress_count) start = datetime.utcnow() build_until = start + timedelta( seconds=self.queue_build_cap) if self.keep_connecting: self.logger.info('Obtained %d %s client%sto `%s` for `%s`', self.queue.maxsize, self.conn_type, suffix, self.address, self.conn_name) else: self.logger.info('Skipped building a queue to `%s` for `%s`', self.address, self.conn_name) # Ok, got all the connections return except KeyboardInterrupt: self.keep_connecting = False def _spawn_add_client_func_no_lock(self, count): for x in range(count): gevent.spawn(self.add_client_func) self.in_progress_count += 1 def _spawn_add_client_func(self, count=1): """ Spawns as many greenlets to populate the connection queue as there are free slots in the queue available. """ with self.lock: if self.queue.full(): logger.info('Queue already full (c:%d) (%s %s)', count, self.address, self.conn_name) return self._spawn_add_client_func_no_lock(count) def decr_in_progress_count(self): with self.lock: self.in_progress_count -= 1 def build_queue(self): """ Spawns greenlets to populate the queue and waits up to self.queue_build_cap seconds until the queue is full. If it never is, raises an exception stating so. """ self._spawn_add_client_func(self.queue.maxsize) # Build the queue in background gevent.spawn(self._build_queue)
class APIRetreiver(object): def __init__(self, config, **options): if not isinstance(config, dict): raise TypeError("Expected a dict as config, got {}".format( type(config))) self.api_host = config.get('api_host') self.api_version = config.get('api_version') self.api_key = config.get('api_key') if 'api_extra_params' in options: self._extra = options.get('api_extra_params') self.tender_queue = Queue(maxsize=config.get('queue_max_size', 250)) self.filter_callback = options.get('filter_callback', lambda x: x) self.forward_worker_dead = Event() self.forward_worker_dead.set() self.backward_worker_dead = Event() self.backward_worker_dead.set() self._init_clients() def _init_clients(self): logger.info('Sync: Init clients') self.forward_client = APICLient(self.api_key, self.api_host, self.api_version) self.backward_client = APICLient(self.api_key, self.api_host, self.api_version) self.origin_cookie = self.forward_client.session.cookies self.backward_client.session.cookies = self.origin_cookie def _get_sync_point(self): logger.info('Sync: initializing sync') forward = {'feed': 'changes'} backward = {'feed': 'changes', 'descending': '1'} if getattr(self, '_extra', ''): [x.update(self._extra) for x in [forward, backward]] r = self.backward_client.get_tenders(backward) backward['offset'] = r['next_page']['offset'] forward['offset'] = r['prev_page']['offset'] logger.error(forward) self.tender_queue.put(filter(self.filter_callback, r['data'])) logger.info('Sync: initial sync params forward: ' '{}, backward: {}'.format(forward, backward)) return forward, backward def _start_sync_workers(self): forward, backward = self._get_sync_point() self.workers = [ gevent.spawn(self._forward_worker, forward), gevent.spawn(self._backward_worker, backward), ] def _forward_worker(self, params): worker = "Forward worker:" logger.info('{} starting'.format(worker)) r = self.forward_client.get_tenders(params) if self.forward_client.session.cookies != self.origin_cookie: raise LBMismatchError try: while True: try: while r['data']: try: self.tender_queue.put( filter(self.filter_callback, r['data'])) except Full: while self.tender_queue.full(): gevent.sleep(QUEUE_FULL_DELAY) self.tender_queue.put( filter(self.filter_callback, r['data'])) params['offset'] = r['next_page']['offset'] r = self.forward_client.get_tenders(params) if self.forward_client.session.cookies != self.origin_cookie: raise LBMismatchError if r['data']: gevent.sleep(FORWARD_WORKER_SLEEP) logger.warn('{} got empty listing. Sleep'.format(worker)) gevent.sleep(ON_EMPTY_DELAY) except LBMismatchError: logger.info('LB mismatch error on backward worker') self.reinit_clients.set() except Exception as e: logger.error("{} down! Error: {}".format(worker, e)) self.forward_worker_dead.set() else: logger.error("{} finished.".format(worker)) def _backward_worker(self, params): worker = "Backward worker: " logger.info('{} staring'.format(worker)) try: while True: try: r = self.backward_client.get_tenders(params) if not r['data']: logger.debug( '{} empty listing..exiting'.format(worker)) break gevent.sleep(BACKWARD_WOKER_DELAY) if self.backward_client.session.cookies != self.origin_cookie: raise LBMismatchError try: self.tender_queue.put( filter(self.filter_callback, r['data'])) except Full: logger.error('full queue') while self.tender_queue.full(): gevent.sleep(QUEUE_FULL_DELAY) self.tender_queue.put( filter(self.filter_callback, r['data'])) params['offset'] = r['next_page']['offset'] except LBMismatchError: logger.info('{} LB mismatch error'.format(worker)) if not self.reinit_clients.is_set(): self.reinit_clients.set() except Exception as e: logger.error("{} down! Error: {}".format(worker, e)) self.forward_worker_dead.set() else: logger.error("{} finished.".format(worker)) def _restart_workers(self): self._init_clients() gevent.killall(self.workers) self._start_sync_workers() return self.workers def get_tenders(self): self._start_sync_workers() forward, backward = self.workers try: while True: if self.tender_queue.empty(): gevent.sleep(EMPTY_QUEUE_DELAY) if (forward.dead or forward.ready()) or \ (backward.dead and not backward.successful()): forward, backward = self._restart_workers() while not self.tender_queue.empty(): yield self.tender_queue.get() except Exception as e: logger.error(e)
class ConnectionQueue(object): """ Holds connections to resources. Each time it's called a connection is fetched from its underlying queue assuming any connection is still available. """ def __init__(self, pool_size, queue_build_cap, conn_name, conn_type, address, add_client_func): self.queue = Queue(pool_size) self.queue_build_cap = queue_build_cap self.conn_name = conn_name self.conn_type = conn_type self.address = address self.add_client_func = add_client_func self.keep_connecting = True self.logger = logging.getLogger(self.__class__.__name__) def __call__(self): return _Connection(self.queue, self.conn_name) def put_client(self, client): self.queue.put(client) self.logger.info('Added `%s` client to %s (%s)', self.conn_name, self.address, self.conn_type) def _build_queue(self): start = datetime.utcnow() build_until = start + timedelta(seconds=self.queue_build_cap) suffix = 's ' if self.queue.maxsize > 1 else ' ' try: while self.keep_connecting and not self.queue.full(): gevent.sleep(0.5) now = datetime.utcnow() self.logger.info('%d/%d %s clients obtained to `%s` (%s) after %s (cap: %ss)', self.queue.qsize(), self.queue.maxsize, self.conn_type, self.address, self.conn_name, now - start, self.queue_build_cap) if now >= build_until: # Log the fact that the queue is not full yet self.logger.warn('Built %s/%s %s clients to `%s` within %s seconds, sleeping until %s (UTC)', self.queue.qsize(), self.queue.maxsize, self.conn_type, self.address, self.queue_build_cap, datetime.utcnow() + timedelta(seconds=self.queue_build_cap)) # Sleep for a predetermined time gevent.sleep(self.queue_build_cap) # Spawn additional greenlets to fill up the queue self._spawn_add_client_func(self.queue.maxsize - self.queue.qsize()) start = datetime.utcnow() build_until = start + timedelta(seconds=self.queue_build_cap) if self.keep_connecting: self.logger.info('Obtained %d %s client%sto `%s` for `%s`', self.queue.maxsize, self.conn_type, suffix, self.address, self.conn_name) else: self.logger.info('Skipped building a queue to `%s` for `%s`', self.address, self.conn_name) # Ok, got all the connections return except KeyboardInterrupt: self.keep_connecting = False def _spawn_add_client_func(self, count): """ Spawns as many greenlets to populate the connection queue as there are free slots in the queue available. """ for x in range(count): gevent.spawn(self.add_client_func) def build_queue(self): """ Spawns greenlets to populate the queue and waits up to self.queue_build_cap seconds until the queue is full. If it never is, raises an exception stating so. """ self._spawn_add_client_func(self.queue.maxsize) # Build the queue in background gevent.spawn(self._build_queue)
class GoGame(game_pb2_grpc.GameServicer): def finishNewThread(self): self.finish = 0 def __init__(self, maxsize=10): self.finish = 1 self.connectionTime = 0 self._players = {} self.gc_pools = Queue(maxsize=maxsize) self.greenlets = [] self.start_console() self.lock = False # def heartBeat(): # while(self.finish==1): # #lock this block # #connection is broken # if (self.connectionTime > 5): # self.lock = True # #clear the board now # for gcItem in self._players.values(): # gcItem.reqChan.put_nowait("clear_board") # while gcItem.respChan.empty(): # sleep(0) # gcItem.respChan.get() # #free all game contexts # keys = [] # for playerId in self._players.keys(): # keys.append(playerId) # for playerId in keys: # self.gc_pools.put_nowait(self._players.pop(playerId)) # print("connection out of time, free all gc") # self.connectionTime = -1 # self.lock = False #calculate for the connection time # self.connectionTime = self.connectionTime + 1 # sleep(1) # try: # self.newThread = threading.Thread(target=heartBeat) # self.newThread.start() # except: # print("Error: unable to start thread") #start the console module def _start_console(self): #create a game context first #the gc is just the game context below gc = GameContext() #add the game context to the pool( 10 gc is allowed to add to the pool in default) self.gc_pools.put_nowait(gc) #define 2 functions here #the console of gc is the GoConsoleGTP def actor(batch): return gc.console.actor(batch) #prompt a command of train #now I consider the batch as a kind of operaion def train(batch): gc.console.prompt("DF Train> ", batch) #the first gc is just the game context below while the second one #is the game context in source code gc.console.evaluator.setup(sampler=env["sampler"], mi=mi) gc.console.GC.reg_callback_if_exists("actor_black", actor) gc.console.GC.reg_callback_if_exists("human_actor", gc.human_actor) gc.console.GC.reg_callback_if_exists("train", train) gc.console.GC.start() gc.console.GC.GC.getClient().setRequest( mi["actor"].step, -1, env['game'].options.resign_thres, -1) gc.console.evaluator.episode_start(0) while True: gc.console.GC.run() if gc.console.exit: break gc.console.GC.stop() def start_console(self): while not self.gc_pools.full(): g = gevent.spawn(self._start_console) g.start() gevent.sleep(0) self.greenlets.append(g) def stop_console(self): gevent.killall(self.greenlets, timeout=10) #one game context is related to one game #the function here will return the protobuf #the function like clear board, play etc., will block here until the response channel is not empty def NewGC(self, request, context): # while (self.lock): # pass #offered by the client playerId = request.id if not playerId in self._players: if self.gc_pools.empty(): return google_dot_rpc_dot_status__pb2.Status( code=google_dot_rpc_dot_code__pb2.RESOURCE_EXHAUSTED, message="no more gc is available") self._players[playerId] = self.gc_pools.get() return google_dot_rpc_dot_status__pb2.Status( code=google_dot_rpc_dot_code__pb2.OK) def FreeGC(self, request, context): # while (self.lock): # pass playerId = request.id if playerId not in self._players: return google_dot_rpc_dot_status__pb2.Status( code=google_dot_rpc_dot_code__pb2.NOT_FOUND) self.gc_pools.put_nowait(self._players.pop(playerId)) return google_dot_rpc_dot_status__pb2.Status( code=google_dot_rpc_dot_code__pb2.OK) def ClearBoard(self, request, context): # while (self.lock): # pass gc = self._players[request.id] #now the game will send message to the game context`s channel, # then those contexts will deal with them orderly gc.reqChan.put_nowait("clear_board") while gc.respChan.empty(): gevent.sleep(0) debug_print("Clear board for player {id}".format(id=request.id)) return gc.respChan.get() def Play(self, request, context): # while (self.lock): # pass gc = self._players[request.player.id] try: gc.action = gc.console.move2action(request.move) except ValueError as e: return game_pb2.Reply(status=google_dot_rpc_dot_status__pb2.Status( code=google_dot_rpc_dot_code__pb2.INVALID_ARGUMENT, message=str(e))) gc.reqChan.put_nowait("play") while gc.respChan.empty(): gevent.sleep(0) return gc.respChan.get() def GenMove(self, request, context): # while (self.lock): # pass gc = self._players[request.id] gc.reqChan.put_nowait("genmove") #until the gc make response this object will be blocked while gc.respChan.empty(): gevent.sleep(0) return gc.respChan.get() def Pass(self, request, context): # while (self.lock): # pass gc = self._players[request.id] gc.reqChan.put_nowait("pass") while gc.respChan.empty(): gevent.sleep(0) return gc.respChan.get() def Resign(self, request, context): # while (self.lock): # pass gc = self._players[request.id] gc.reqChan.put_nowait("resign") print("put") while gc.respChan.empty(): gevent.sleep(0) print("return value") return gc.respChan.get() def HeartBeat(self, request, context): # while (self.lock): # pass self.connectionTime = 0 return game_pb2.BeatReply(beatReply=1)
class FrontendHandleBase(object): """ This base class for frontend handle """ def __init__(self, handle_id, session, plugin, opaque_id=None): self.handle_id = handle_id self.opaque_id = opaque_id self._session = session self._plugin = plugin self._has_destroy = False self.created = time.time() self.plugin_package_name = plugin.get_package() self._async_message_queue = Queue(maxsize=1024) self._async_message_greenlet = gevent.spawn( self._async_message_handler_routine) def detach(self): if self._has_destroy: return self._has_destroy = True # stop async message greenlet if not self._async_message_queue.full(): self._async_message_queue.put(stop_message) self._async_message_greenlet = None # send detach event event = create_janus_msg('detached', self._session.session_id) event['sender'] = self.handle_id if self.opaque_id: event['opaque_id'] = self.opaque_id self._session.notify_event(event) log.info('handle {} is detach from plugin {}'.format( self.handle_id, self.plugin_package_name)) def has_destroy(self): return self._has_destroy def handle_hangup(self): raise JanusCloudError('hangup not support\'hangup\'', JANUS_ERROR_MISSING_REQUEST) def handle_message(self, transaction, body, jsep=None): raise JanusCloudError('message not support\'message\'', JANUS_ERROR_PLUGIN_MESSAGE) def handle_trickle(self, candidate=None, candidates=None): raise JanusCloudError('hangup not support\'trickle\'', JANUS_ERROR_MISSING_REQUEST) def _enqueue_async_message(self, transaction, body, jsep=None): self._async_message_queue.put_nowait((transaction, body, jsep)) def _async_message_handler_routine(self): while not self._has_destroy: msg = self._async_message_queue.get() if self._has_destroy or msg == stop_message: return try: transaction, body, jsep = msg self._handle_async_message(transaction, body, jsep) except Exception: log.exception( 'Error when handle async message for handle {}'.format( self.handle_id)) def _handle_async_message(self, transaction, body, jsep): """ need to override by subclass """ raise JanusCloudError('async message handler not support\'message\'', JANUS_ERROR_PLUGIN_MESSAGE) def _push_plugin_event(self, data, jsep=None, transaction=None): params = dict() params['plugindata'] = { 'plugin': self.plugin_package_name, 'data': data } if jsep: params['jsep'] = jsep self._push_event('event', transaction, **params) def _push_event(self, method, transaction=None, **kwargs): if self._has_destroy: return event = create_janus_msg(method, self._session.session_id, transaction, **kwargs) event['sender'] = self.handle_id if self.opaque_id: event['opaque_id'] = self.opaque_id self._session.notify_event(event)
class WFEngineDaemon(GenericDaemon): def __init__(self, **kwargs): from gevent.queue import Queue super(WFEngineDaemon, self).__init__(**kwargs) self.task_model = get_model('workflow_task') self.limit = kwargs.get("query_limit", 10) self.daemon_id = kwargs.get("daemon_id", get_unique_id()) #worker self.worker_number = kwargs.get("worker", 4) self.workers = {} self.queue_limit = kwargs.get("queue_size", 10) self.queue = Queue(self.queue_limit) self.status = None self.is_worker_started = False self.tid = 0 self.register_request('pause', usage="Pause to load new task.") self.register_request('resume', usage="Resume to load new task.") self.register_request('worker', usage="Show worker information.") self.register_request('reset', inner=False) self.mainLoopCommands = ["start_workers"] def get_server_info(self): info = [] info.append("%s" % __daemon_name__) info.append("- version: %s" % __daemon_version__) info.append("- port: %s" % self.port or __daemon_port__) info.append("- instance: %s" % self.daemon_id) return info def get_worker_name(self, i): a, b = i//6+1, i % 6 name = "%s%d" % (__WORKER_NAMES__[b], a) while self.workers.has_key(name): a = a + 1 name = "%s%d" % (__WORKER_NAMES__[b], a) return name def loader(self): import gevent while True: if self.status == STOPPED: break if self.status == RUNNING: # load task from database or redis # load task from database: self.load_ready_tasks(self.queue) gevent.sleep(0) else: gevent.sleep(0.5) self.prints(">>> Loader is stopped at %s" % self.gettimestamp()) return def worker(self, name, index): import gevent from random import randint work_status = RUNNING self.workers[name] = Worker(name, RUNNING, index) w = self.workers[name] while True: if w.status == PAUSED: if self.status == RUNNING: w.status == RUNNING if w.status == RUNNING: if not self.queue.empty(): task = self.queue.get() self.prints('>>> %s got task %s ' % (name, task[0])) self.async_deliver(task[1]) else: if self.status == PAUSED: self.prints(">>> %s is paused at %s" % (name, self.gettimestamp())) w.status = PAUSED if w.status == STOPPED: break gevent.sleep(0) del self.workers[name] self.prints(">>> %s is stopped at %s" % (name, self.gettimestamp())) self.worker_number = len(self.workers) if self.worker_number > 1: self.prints(">>> still have %d workers at server." % self.worker_number) elif self.worker_number == 1: self.prints(">>> only one worker at server." ) else: self.prints(">>> no running worker at server.") def load_ready_tasks(self, queue): from uliweb.orm import Begin, Commit, Rollback, Reset from gevent.queue import Full import gevent if self.queue.full(): gevent.sleep(0.3) return Begin() try: cond = self.task_model.c.state == 2 # READY cond = (self.task_model.c.async_status == 0 or self.task_model.c.async_status == None) & cond query = self.task_model.filter(cond) query = query.order_by(self.task_model.c.async_deliver_date.asc()) query = query.limit(self.limit) isFull = False isEmpty = True for item in query: self.tid = self.tid + 1 try: queue.put_nowait([self.tid, item.id]) item.update(async_status=1) item.save() isEmpty = False except Full: isFull = True break Commit() if isEmpty and self.debug: self.prints(">>> [DEBUG] No tasks found %s" % self.gettimestamp()) if isFull: gevent.sleep(0.3) if isEmpty: gevent.sleep(0.3) except: Rollback() import traceback traceback.print_exc() def startMainLoop1(self): self.start_workers() self.prints(">>> MainLoop is stoped at %s" % self.gettimestamp()) def mainLoop(self, cmd=None): import gevent if self.mainLoopCommands: cmd = self.mainLoopCommands.pop() if hasattr(self, "do_%s" % cmd): gevent.spawn(getattr(self, "do_%s" % cmd)) #self.prints("mainLoop ..") def do_start_workers(self): import gevent self.prints(">>> Workers are startting....") self.status = RUNNING # one loader greentlets = [gevent.spawn(self.loader)] # some workers for i in range(1, self.worker_number+1): name = self.get_worker_name(i) greentlets.append(gevent.spawn(self.worker, name, i)) gevent.joinall(greentlets) def handle_shutdown(self, req): self.prints(">>> Worker stopping....") self.status = STOPPED return super(WFEngineDaemon, self).handle_shutdown(req) def handle_pause(self, req): self.status = PAUSED self.prints(">>> Loader is paused at %s" % self.gettimestamp()) return self.create_response(True, "The loader is pausing.") def handle_resume(self, req): self.status = RUNNING return self.create_response(True, "The loader is resuming.") def handle_worker(self, req): subcmd = req.msg data = req.data if subcmd == "add": import gevent if data and data.isdigit(): addcount = int(data) else: addcount = 1 msg = [] for i in range(1, addcount+1): self.worker_number = self.worker_number + 1 name = self.get_worker_name(self.worker_number) greenlet = gevent.spawn(self.worker, name, self.worker_number) msg.append("new worker %s added" % name) return self.create_response(True, "\n".join(msg)) if subcmd == "show": if data: if self.workers.has_key(data): w = self.workers[name] msg = "%10s %10s" % (w.index, w.name, w.status) return self.create_response(True, msg) else: return self.create_response(False, "cannot find worker %s" % data) if subcmd == "del": if data: if data == "all": msg = [] for name in self.workers: w = self.workers[name] w.status = STOPPED msg.append("worker %s was stopped." % name) return self.create_response(True, "\n".join(msg)) if self.workers.has_key(data): w = self.workers[data] w.status = STOPPED return self.create_response(True, "worker %s was stopped." % data) return self.create_response(False, "cannot find worker %s" % data) msg = [] for name in sorted(self.workers): w = self.workers[name] msg.append("%10s %10s" % (w.name, w.status)) return self.create_response(True, "\n".join(msg)) def handle_reset(self, req): Task = self.task_model count = 0 for item in Task.filter(Task.c.async_status == 1): item.async_status = None item.save() count = count + 1 return self.create_response(True, "reset %d tasks in database." % count) def async_deliver(self, task_id): from redbreast.serializable import Workflow, Task task_obj = self.task_model.get(task_id) if task_obj.state == 2: #READY wf_id = task_obj._workflow_ workflow = Workflow.load(wf_id, operator=task_obj._modified_user_) task = workflow.get_task(task_obj.uuid) self.prints("------------------------------------------------") self.prints("spec %s %s(%s)" % (workflow.get_spec_name(), task.get_name(), task.get_spec_name())) message = task.deliver_msg next_tasks = task.next_tasks workflow.deliver(task_obj.uuid, message=message, next_tasks=next_tasks, async=False) task_obj.update(async_status=0) task_obj.save()
class BackendHandle(object): """ This backend handle represents a Janus handle """ def __init__(self, handle_id, plugin_package_name, session, opaque_id=None, handle_listener=None): self.handle_id = handle_id self.plugin_package_name = plugin_package_name self.opaque_id = opaque_id self._session = session self._has_detach = False self._handle_listener = handle_listener self._async_event_queue = Queue(maxsize=1024) self._async_event_greenlet = gevent.spawn(self._async_event_handler_routine) def detach(self): """ detach this handle from the session return: no value note: no exception would be raised """ if self._has_detach: return self._has_detach = True # stop async event greenlet if not self._async_event_queue.full(): self._async_event_queue.put(stop_message) self._async_event_greenlet = None if self._session: self._session.on_handle_detached(self.handle_id) try: detach_message = create_janus_msg('detach', handle_id=self.handle_id) self._session.send_request(detach_message) except Exception: log.exception('Detach backend handle {} error'.format(self.handle_id)) self._session = None def send_message(self, body, jsep=None): if self._has_detach: raise JanusCloudError('backend handle {} has been destroyed'.format(self.handle_id), JANUS_ERROR_PLUGIN_DETACH) params = dict() params['body'] = body if jsep: params['jsep'] = jsep message = create_janus_msg('message', handle_id=self.handle_id, **params) response = self._session.send_request(message) if response['janus'] == 'event' or response['janus'] == 'success': data = response['plugindata']['data'] reply_jsep = response.get('jsep') return data, reply_jsep elif response['janus'] == 'error': raise JanusCloudError(response['error']['reason'], response['error']['code']) else: raise JanusCloudError( 'unknown backend response {}'.format(response), JANUS_ERROR_BAD_GATEWAY) def send_trickle(self, candidate=None, candidates=None): if self._has_detach: raise JanusCloudError('backend handle {} has been destroyed'.format(self.handle_id), JANUS_ERROR_PLUGIN_DETACH) if candidate is None and candidates is None: raise JanusCloudError('Missing mandatory element (candidate|candidates)', JANUS_ERROR_MISSING_MANDATORY_ELEMENT) if candidate and candidates: raise JanusCloudError('Can\'t have both candidate and candidates', JANUS_ERROR_INVALID_JSON) params = {} if candidate: params['candidate'] = candidate if candidates: params['candidates'] = candidates trickle_msg = create_janus_msg('trickle', handle_id=self.handle_id, **params) response = self._session.send_request(trickle_msg, ignore_ack=False) if response['janus'] == 'ack': pass # successful elif response['janus'] == 'error': raise JanusCloudError(response['error']['reason'], response['error']['code']) else: raise JanusCloudError( 'unknown backend response {}'.format(response), JANUS_ERROR_BAD_GATEWAY) def send_hangup(self): if self._has_detach: raise JanusCloudError('backend handle {} has been destroyed'.format(self.handle_id), JANUS_ERROR_PLUGIN_DETACH) hangup_msg = create_janus_msg('hangup', handle_id=self.handle_id) response = self._session.send_request(hangup_msg) if response['janus'] == 'success': pass # successful elif response['janus'] == 'error': raise JanusCloudError(response['error']['reason'], response['error']['code']) else: raise JanusCloudError( 'unknown backend response {}'.format(response), JANUS_ERROR_BAD_GATEWAY) def on_async_event(self, event_msg): if not self._async_event_queue.full(): self._async_event_queue.put(event_msg) else: # drop the event log.error("backend handle {} async event queue is full, drop the receiving event".format(self.handle_id)) def on_close(self): if self._has_detach: return self._has_detach = True # stop async event greenlet if not self._async_event_queue.full(): self._async_event_queue.put(stop_message) self._async_event_greenlet = None self._session = None if self._handle_listener: try: self._handle_listener.on_close(self.handle_id) except Exception: log.exception('on_close() exception for backend handle {}'.format(self.handle_id)) def _async_event_handler_routine(self): while not self._has_detach: event_msg = self._async_event_queue.get() if self._has_detach or event_msg == stop_message: return try: if self._handle_listener: self._handle_listener.on_async_event(event_msg) except Exception: log.exception('Error when handle async event for backend handle {}'.format(self.handle_id))
class DriverPool(object): """ Create a pool of available Selenium containers for processing. Args: size (int): maximum concurrent tasks. Must be at least ``2``. driver_cls (WebDriver): driver_cls_args (tuple): driver_cls_kw (dict): use_proxy (bool): factory (:obj:`~selenium_docker.base.ContainerFactory`): name (str): logger (:obj:`logging.Logger`): Example:: pool = DriverPool(size=2) urls = [ 'https://google.com', 'https://reddit.com', 'https://yahoo.com', 'http://ksl.com', 'http://cnn.com' ] def get_title(driver, url): driver.get(url) return driver.title for result in pool.execute(get_title, urls): print(result) """ INNER_THREAD_SLEEP = 0.5 """float: essentially our polling interval between tasks and checking when tasks have completed. """ PROXY_CLS = SquidProxy """:obj:`~selenium_docker.proxy.AbstractProxy`: created for the pool when ``use_proxy=True`` during pool instantiation. """ def __init__(self, size, driver_cls=ChromeDriver, driver_cls_args=None, driver_cls_kw=None, use_proxy=True, factory=None, name=None, logger=None): self.size = max(2, size) self.name = name or gen_uuid(6) self.factory = factory or ContainerFactory.get_default_factory() self.logger = logger or getLogger( '%s.DriverPool.%s' % (__name__, self.name)) self._driver_cls = driver_cls self._driver_cls_args = driver_cls_args or tuple() self._driver_cls_kw = driver_cls_kw or dict() self._drivers = Queue(maxsize=self.size) # post init inspections if not hasattr(self._driver_cls, 'CONTAINER'): raise DriverPoolValueError('driver_cls must extend DockerDriver') if not isiterable(self._driver_cls_args): raise DriverPoolValueError( '%s is not iterable' % self._driver_cls_args) if not isinstance(self._driver_cls_kw, Mapping): raise DriverPoolValueError( '%s is not a valid mapping' % self._driver_cls_kw) # determine proxy usage self.proxy = None self._use_proxy = use_proxy # type: bool # deferred instantiation self._pool = None # type: Pool self._results = None # type: Queue self._tasks = None # type: JoinableQueue self._processing = False # type: bool self.__feeder_green = None # type: gevent.Greenlet def __repr__(self): return '<DriverPool-%s(size=%d,driver=%s,proxy=%s,async=%s)>' % ( self.name, self.size, self._driver_cls.BROWSER, self._use_proxy, self.is_async) def __iter__(self): return self.results(block=self.is_async) def __del__(self): try: self.close() except Exception as e: if hasattr(self, 'logger'): self.logger.exection(e, exc_info=False) @property def is_processing(self): """bool: whether or not we're currently processing tasks. """ return self._processing @property def is_async(self): """bool: returns True when asynchronous processing is happening. """ return self.__feeder_green is not None def __bootstrap(self): """ Prepare this driver pool instance to batch execute task items. """ if self.is_processing: # cannot run two executions simultaneously raise DriverPoolRuntimeException( 'cannot bootstrap pool, already running') if self._results and self._results.qsize(): # pragma: no cover self.logger.debug('pending results being discarded') if self._tasks and self._tasks.qsize(): # pragma: no cover self.logger.debug('pending tasks being discarded') if self._pool: # pragma: no cover self.logger.debug('killing processing pool') self._pool.join(timeout=10.0) self._pool.kill() self._pool = None if self._use_proxy and not self.proxy: # defer proxy instantiation -- since spinning up a squid proxy # docker container is surprisingly time consuming. self.logger.debug('bootstrapping squid proxy') self.proxy = self.PROXY_CLS(factory=self.factory) self.logger.debug('bootstrapping pool processing') self._processing = True self._results = Queue() self._tasks = JoinableQueue() self._load_drivers() # create our processing pool with headroom over the number of drivers # requested for this processing pool. self._pool = Pool(size=self.size + math.ceil(self.size * 0.25)) def __cleanup(self, force=False): """ Stop and remove the web drivers and their containers. This function should not remove pending tasks or results. It should be possible to cleanup all the external resources of a driver pool and still extract the results of the work that was completed. Raises: DriverPoolRuntimeException: when attempting to cleanup an environment while processing is still happening, and forcing the cleanup is set to ``False``. SeleniumDockerException: when a driver instance or container cannot be closed properly. Returns: None """ if self.is_processing and not force: # pragma: no cover raise DriverPoolRuntimeException( 'cannot cleanup driver pool while executing') self._processing = False squid = None # type: gevent.Greenlet error = None # type: SeleniumDockerException if self.proxy: self.logger.debug('closing squid proxy') squid = gevent.spawn(self.proxy.quit) if self._pool: # pragma: no cover self.logger.debug('emptying task pool') if not force: self._pool.join(timeout=10.0) self._pool.kill(block=False, timeout=10.0) self._pool = None self.logger.debug('closing all driver containers') while not self._drivers.empty(): d = self._drivers.get(block=True) try: d.quit() except SeleniumDockerException as e: # pragma: no cover self.logger.exception(e, exc_info=True) if not force: error = e if self.proxy: squid.join() self.proxy = None if error: # pragma: no cover raise error def _load_driver(self, and_add=True): """ Load a single web driver instance and container. """ args = self._driver_cls_args kw = dict(self._driver_cls_kw) kw.update({ 'proxy': self.proxy, 'factory': self.factory, }) driver = self._driver_cls(*args, **kw) if and_add: self._drivers.put(driver) return driver def _load_drivers(self): """ Load the web driver instances and containers. Raises: DriverPoolRuntimeException: when the requested number of drivers for the given pool size cannot be created for some reason. Returns: None """ if not self._drivers.empty(): # pragma: no cover return threads = [] for o in range(self.size): self.logger.debug('creating driver %d of %d', o + 1, self.size) thread = gevent.spawn(self._load_driver) threads.append(thread) for t in reversed(threads): t.join() if not self._drivers.full(): raise DriverPoolRuntimeException( 'unable to fulfill required concurrent drivers, %d of %d' % ( self._drivers.qsize(), self.size)) def _recycle_driver(self, driver): if not driver: return try: driver.quit() except Exception as e: self.logger.exception(e, exc_info=True) # do NOT add the new driver container to the drivers queue, # instead this will be handled in the recycle logic that requested # the driver in the first place. Instead of returning the one it # received this "new" instance will be put in its placed. print('RECYCLED!!!!!!') return self._load_driver(and_add=False) def add_async(self, *items): """ Add additional items to the asynchronous processing queue. Args: items (list(Any)): list of items that need processing. Each item is applied one at a time to an available driver from the pool. Raises: StopIteration: when all items have been added. """ if len(items) == 1 and isinstance(items[0], list): items = iter(items[0]) if not items: raise DriverPoolValueError( 'cannot add items with value: %s' % str(items)) item_count = count(items) self.logger.debug('adding %d additional items to tasks', item_count) for o in items: self._tasks.put(o) def close(self): """ Force close all the drivers and cleanup their containers. Returns: None """ self.__cleanup(force=True) def execute(self, fn, items, preserve_order=False, auto_clean=True, no_wait=False): """ Execute a fixed function, blocking for results. Args: fn (Callable): function that takes two parameters, ``driver`` and ``task``. items (list(Any)): list of items that need processing. Each item is applied one at a time to an available driver from the pool. preserve_order (bool): should the results be returned in the order they were supplied via ``items``. It's more performant to allow results to return in any order. auto_clean (bool): cleanup docker containers after executing. If multiple processing tasks are going to be used, it's more performant to leave the containers running and reuse them. no_wait (bool): forgo a small sleep interval between finishing a task and putting the driver back in the available drivers pool. Yields: results: the result for each item as they're finished. """ def worker(o): job_num, item = o self.logger.debug('doing work on item %d' % job_num) driver = self._drivers.get(block=True) ret_val = fn(driver, item) if not no_wait: gevent.sleep(self.INNER_THREAD_SLEEP) self._drivers.put(driver) return ret_val if self.__feeder_green: raise DriverPoolRuntimeException( 'cannot perform a blocking execute while async processing') self.__bootstrap() self.logger.debug('starting sync processing') if preserve_order: ittr = self._pool.imap else: ittr = self._pool.imap_unordered self.logger.debug('yielding processed results') for o in ittr(worker, enumerate(items)): self._results.put(o) self._results.put(StopIteration) self.logger.debug('stopping sync processing') if auto_clean: self.logger.debug('auto cleanup pool environment') self.__cleanup(force=True) return self.results(block=False) def execute_async(self, fn, items=None, callback=None, catch=(WebDriverException,), requeue_task=False): """ Execute a fixed function in the background, streaming results. Args: fn (Callable): function that takes two parameters, ``driver`` and ``task``. items (list(Any)): list of items that need processing. Each item is applied one at a time to an available driver from the pool. callback (Callable): function that takes a single parameter, the return value of ``fn`` when its finished processing and has returned the driver to the queue. catch (tuple[Exception]): tuple of Exception classes to catch during task execution. If one of these Exception classes is caught during ``fn`` execution the driver that crashed will attempt to be recycled. requeue_task (bool): in the event of an Exception being caught should the task/item that was being worked on be re-added to the queue of items being processed. Raises: DriverPoolValueError: if ``callback`` is not ``None`` or ``callable``. Returns: None """ def worker(fn, task): ret_val = None async_task_id = gen_uuid(12) self.logger.debug('starting async task %s', async_task_id) driver = self._drivers.get(block=True) if isinstance(driver, Exception): raise driver try: ret_val = fn(driver, task) except catch as e: self.logger.exception('hihi') if self.is_processing: driver = self._recycle_driver(driver) if requeue_task: self._tasks.put(task) finally: self._results.put(ret_val) self._drivers.put(driver) gevent.sleep(self.INNER_THREAD_SLEEP) return ret_val def feeder(): self.logger.debug('starting async feeder thread') while True: while not self._tasks.empty(): task = self._tasks.get() if self._pool is None: break self._pool.apply_async( worker, args=(fn, task,), callback=greenlet_callback) gevent.sleep(self.INNER_THREAD_SLEEP) if self._pool is None and not self.is_processing: break return if callback is None: def logger(value): self.logger.debug('%s', value) callback = logger def real_callback(cb, value): if isinstance(value, gevent.GreenletExit): raise value else: cb(value) greenlet_callback = partial(real_callback, callback) for f in [fn, callback]: if not callable(f): raise DriverPoolValueError( 'cannot use %s, is not callable' % callback) self.logger.debug('starting async processing') self.__bootstrap() if not self.__feeder_green: self.__feeder_green = gevent.spawn(feeder) if items: self.add_async(*items) def quit(self): """ Alias for :func:`~DriverPool.close()`. Included for consistency with driver instances that generally call ``quit`` when they're no longer needed. Returns: None """ if self.__feeder_green: return self.stop_async() return self.close() def results(self, block=True): """ Iterate over available results from processed tasks. Args: block (bool): when ``True``, block this call until all tasks have been processed and all results have been returned. Otherwise this will continue indefinitely while tasks are dynamically added to the async processing queue. Yields: results: one result at a time as they're finished. Raises: StopIteration: when the processing is finished. """ est_size = self._results.qsize() self.logger.debug('there are an estimated %d results', est_size) if block: self.logger.debug('blocking for results to finish processing') while self.is_processing: while not self._results.empty(): yield self._results.get() gevent.sleep(self.INNER_THREAD_SLEEP) if self._tasks.empty() and self._results.empty(): break raise StopIteration else: if est_size > 0: self.logger.debug('returning as many results as have finished') self._results.put(StopIteration) for result in self._results: yield result def stop_async(self, timeout=None, auto_clean=True): """ Stop all the async worker processing from executing. Args: timeout (float): number of seconds to wait for pool to finish processing before killing and closing out the execution. auto_clean (bool): cleanup docker containers after executing. If multiple processing tasks are going to be used, it's more performant to leave the containers running and reuse them. Returns: None """ self.logger.debug('stopping async processing') if self.__feeder_green: self.logger.debug('killing async feeder thread') gevent.kill(self.__feeder_green) self.__feeder_green = None if self._pool: self.logger.debug('joining async pool before kill') self._pool.join(timeout=timeout or 1.0) self._pool.kill(block=False) tasks_count = self._tasks.qsize() self.logger.info('%d tasks remained unprocessed', tasks_count) if auto_clean: self.logger.debug('auto cleanup pool environment') self.__cleanup(force=True)
class WFEngineDaemon(GenericDaemon): def __init__(self, **kwargs): from gevent.queue import Queue super(WFEngineDaemon, self).__init__(**kwargs) self.task_model = get_model('workflow_task') self.limit = kwargs.get("query_limit", 10) self.daemon_id = kwargs.get("daemon_id", get_unique_id()) #worker self.worker_number = kwargs.get("worker", 4) self.workers = {} self.queue_limit = kwargs.get("queue_size", 10) self.queue = Queue(self.queue_limit) self.status = None self.is_worker_started = False self.tid = 0 self.register_request('pause', usage="Pause to load new task.") self.register_request('resume', usage="Resume to load new task.") self.register_request('worker', usage="Show worker information.") self.register_request('reset', inner=False) self.mainLoopCommands = ["start_workers"] def get_server_info(self): info = [] info.append("%s" % __daemon_name__) info.append("- version: %s" % __daemon_version__) info.append("- port: %s" % self.port or __daemon_port__) info.append("- instance: %s" % self.daemon_id) return info def get_worker_name(self, i): a, b = i // 6 + 1, i % 6 name = "%s%d" % (__WORKER_NAMES__[b], a) while self.workers.has_key(name): a = a + 1 name = "%s%d" % (__WORKER_NAMES__[b], a) return name def loader(self): import gevent while True: if self.status == STOPPED: break if self.status == RUNNING: # load task from database or redis # load task from database: self.load_ready_tasks(self.queue) gevent.sleep(0) else: gevent.sleep(0.5) self.prints(">>> Loader is stopped at %s" % self.gettimestamp()) return def worker(self, name, index): import gevent from random import randint work_status = RUNNING self.workers[name] = Worker(name, RUNNING, index) w = self.workers[name] while True: if w.status == PAUSED: if self.status == RUNNING: w.status == RUNNING if w.status == RUNNING: if not self.queue.empty(): task = self.queue.get() self.prints('>>> %s got task %s ' % (name, task[0])) self.async_deliver(task[1]) else: if self.status == PAUSED: self.prints(">>> %s is paused at %s" % (name, self.gettimestamp())) w.status = PAUSED if w.status == STOPPED: break gevent.sleep(0) del self.workers[name] self.prints(">>> %s is stopped at %s" % (name, self.gettimestamp())) self.worker_number = len(self.workers) if self.worker_number > 1: self.prints(">>> still have %d workers at server." % self.worker_number) elif self.worker_number == 1: self.prints(">>> only one worker at server.") else: self.prints(">>> no running worker at server.") def load_ready_tasks(self, queue): from uliweb.orm import Begin, Commit, Rollback, Reset from gevent.queue import Full import gevent if self.queue.full(): gevent.sleep(0.3) return Begin() try: cond = self.task_model.c.state == 2 # READY cond = (self.task_model.c.async_status == 0 or self.task_model.c.async_status == None) & cond query = self.task_model.filter(cond) query = query.order_by(self.task_model.c.async_deliver_date.asc()) query = query.limit(self.limit) isFull = False isEmpty = True for item in query: self.tid = self.tid + 1 try: queue.put_nowait([self.tid, item.id]) item.update(async_status=1) item.save() isEmpty = False except Full: isFull = True break Commit() if isEmpty and self.debug: self.prints(">>> [DEBUG] No tasks found %s" % self.gettimestamp()) if isFull: gevent.sleep(0.3) if isEmpty: gevent.sleep(0.3) except: Rollback() import traceback traceback.print_exc() def startMainLoop1(self): self.start_workers() self.prints(">>> MainLoop is stoped at %s" % self.gettimestamp()) def mainLoop(self, cmd=None): import gevent if self.mainLoopCommands: cmd = self.mainLoopCommands.pop() if hasattr(self, "do_%s" % cmd): gevent.spawn(getattr(self, "do_%s" % cmd)) #self.prints("mainLoop ..") def do_start_workers(self): import gevent self.prints(">>> Workers are startting....") self.status = RUNNING # one loader greentlets = [gevent.spawn(self.loader)] # some workers for i in range(1, self.worker_number + 1): name = self.get_worker_name(i) greentlets.append(gevent.spawn(self.worker, name, i)) gevent.joinall(greentlets) def handle_shutdown(self, req): self.prints(">>> Worker stopping....") self.status = STOPPED return super(WFEngineDaemon, self).handle_shutdown(req) def handle_pause(self, req): self.status = PAUSED self.prints(">>> Loader is paused at %s" % self.gettimestamp()) return self.create_response(True, "The loader is pausing.") def handle_resume(self, req): self.status = RUNNING return self.create_response(True, "The loader is resuming.") def handle_worker(self, req): subcmd = req.msg data = req.data if subcmd == "add": import gevent if data and data.isdigit(): addcount = int(data) else: addcount = 1 msg = [] for i in range(1, addcount + 1): self.worker_number = self.worker_number + 1 name = self.get_worker_name(self.worker_number) greenlet = gevent.spawn(self.worker, name, self.worker_number) msg.append("new worker %s added" % name) return self.create_response(True, "\n".join(msg)) if subcmd == "show": if data: if self.workers.has_key(data): w = self.workers[name] msg = "%10s %10s" % (w.index, w.name, w.status) return self.create_response(True, msg) else: return self.create_response(False, "cannot find worker %s" % data) if subcmd == "del": if data: if data == "all": msg = [] for name in self.workers: w = self.workers[name] w.status = STOPPED msg.append("worker %s was stopped." % name) return self.create_response(True, "\n".join(msg)) if self.workers.has_key(data): w = self.workers[data] w.status = STOPPED return self.create_response( True, "worker %s was stopped." % data) return self.create_response(False, "cannot find worker %s" % data) msg = [] for name in sorted(self.workers): w = self.workers[name] msg.append("%10s %10s" % (w.name, w.status)) return self.create_response(True, "\n".join(msg)) def handle_reset(self, req): Task = self.task_model count = 0 for item in Task.filter(Task.c.async_status == 1): item.async_status = None item.save() count = count + 1 return self.create_response(True, "reset %d tasks in database." % count) def async_deliver(self, task_id): from redbreast.serializable import Workflow, Task task_obj = self.task_model.get(task_id) if task_obj.state == 2: #READY wf_id = task_obj._workflow_ workflow = Workflow.load(wf_id, operator=task_obj._modified_user_) task = workflow.get_task(task_obj.uuid) self.prints("------------------------------------------------") self.prints("spec %s %s(%s)" % (workflow.get_spec_name(), task.get_name(), task.get_spec_name())) message = task.deliver_msg next_tasks = task.next_tasks workflow.deliver(task_obj.uuid, message=message, next_tasks=next_tasks, async=False) task_obj.update(async_status=0) task_obj.save()
class MsgpackEndpoint(object): def __init__(self, mode, conn, conn_error_handle, router=None, timeout=5.0, poolsize=10, chunksize=1024*32, pack_encoding='utf-8', unpack_encoding='utf-8'): self._router = router self._conn_error_handle = conn_error_handle self._mode = mode self._timeout = timeout self._poolsize = poolsize self._sendqueue = Queue(maxsize=poolsize) self._msgpool = dict() self._pack_encoding = pack_encoding self._unpack_encoding = unpack_encoding self._packer = msgpack.Packer(use_bin_type=True, encoding=pack_encoding.encode("utf-8")) self._unpacker = msgpack.Unpacker(encoding=unpack_encoding.encode("utf-8")) self._conn = conn self._chunksize = chunksize self._connecting = False self._msgid = 0 self._reading_worker = gevent.spawn(self._reading) self._sending_worker = gevent.spawn(self._sending) self._reading_worker.start() self._sending_worker.start() def _reading(self): while True: if not self._conn: gevent.sleep(1.0) continue try: data = self._conn.recv(self._chunksize) except socket.timeout as t: continue if not data: logging.warning("connection closed") tmpconn = self._conn self._conn = None self._conn_error_handle.on_conn_error(tmpconn, self, Exception("connection closed")) continue self._unpacker.feed(data) while True: try: msg = self._unpacker.next() except StopIteration as e: break self._parse_msg(msg) def _sending(self): while True: if not self._conn: gevent.sleep(1.0) continue body = self._sendqueue.get() try: self._conn.sendall(body) except Exception as e: logging.warning("RPCClient._sending:error occured. {}".format(e)) tmpconn = self._conn self._conn = None if not self._sendqueue.full(): self._sendqueue.put(body) self._conn_error_handle.on_conn_error(tmpconn, self, e) continue def _parse_msg(self, msg): if (type(msg) != list and type(msg) != tuple) or len(msg) < 3: logging.warn("invalid msgpack-rpc msg. type={}, msg={}".format(type(msg), msg)) tmpconn = self._conn self._conn = None self._conn_error_handle.on_conn_error(tmpconn, self, Exception("invalid msgpack-rpc msg")) return if msg[0] == MSGPACKRPC_RSP and len(msg) == 4 and self._mode & MODECLIENT: (_, msgid, error, result) = msg if msgid not in self._msgpool: logging.warn("unexpected msgid. msgid = {}".format(msgid)) return msgsit = self._msgpool[msgid] del self._msgpool[msgid] msgsit[1] = error msgsit[2] = result msgsit[0].set() elif msg[0] == MSGPACKRPC_REQ and len(msg) == 4 and self._mode & MODESERVER: (_, msgid, method, params) = msg func = self._router.get_call(method) result = None if not func: rsp = (MSGPACKRPC_RSP, msgid, "Method not found: {}".format(method), None) self._sendqueue.put(self._packer.pack(rsp)) return if not hasattr(func, '__call__'): rsp = (MSGPACKRPC_RSP, msgid, "Method is not callable: {}".format(method), None) self._sendqueue.put(self._packer.pack(rsp)) return try: result = func(*params) except Exception as e: rsp = (MSGPACKRPC_RSP, msgid, "{}".format(e), None) self._sendqueue.put(self._packer.pack(rsp)) return rsp = (MSGPACKRPC_RSP, msgid, None, result) self._sendqueue.put(self._packer.pack(rsp)) elif msg[0] == MSGPACKRPC_NOTIFY and len(msg) == 3 and self._mode & MODESERVER: (_, method, params) = msg func = self._router.get_notify(method) if not func: logging.warn("Method not found: {}".format(method)) return if not hasattr(func, '__call__'): logging.warn("Method is not callable: {}".format(method)) return try: func(*params) except Exception as e: logging.warn("Exception: {} in notify {}".format(e, method)) return else: logging.warn("invalid msgpack-rpc msg {}".format(msg)) tmpconn = self._conn self._conn = None self._conn_error_handle.on_conn_error(tmpconn, self, Exception("invalid msgpack-rpc msg")) return def attach_conn(self, conn): if self._conn: return False self._conn = conn return True def call(self, method, *args): if not self._conn: logging.warn("rpc connection closed") if type(method) != str or type(args) != tuple: raise Exception("invalid msgpack-rpc request, type(method)={}, type(args)={}".format(type(method), type(args))) self._msgid += 1 msgid = self._msgid req = (MSGPACKRPC_REQ, msgid, method, args) body = self._packer.pack(req) msgsit = [Event(), None, None] self._msgpool[msgid] = msgsit self._sendqueue.put(body) r = msgsit[0].wait(timeout=self._timeout) if not r: raise Exception("msgpack-rpc call timeout after {} seconds, msgid = {}".format(self._timeout, msgid)) if msgsit[1]: raise Exception("msgpack-rpc call rsp error. {}".format(msgsit[1])) return msgsit[2] def notify(self, method, *args): if not self._conn: logging.warn("rpc connection closed") if type(method) != str or type(args) != tuple: raise Exception("invalid msgpack-rpc request, type(method)={}, type(args)={}".format(type(method), type(args))) notify = (MSGPACKRPC_NOTIFY, method, args) self._sendqueue.put(self._packer.pack(notify)) def close(self): if self._reading_worker: self._reading_worker.kill() self._reading_worker = None if self._sending_worker: self._sending_worker.kill() self._sending_worker = None if self._conn: self._conn = None