class ConnectionQueue(object):
""" Holds connections to resources. Each time it's called a connection is fetched from its underlying queue
assuming any connection is still available.
"""
def __init__(self, pool_size, queue_build_cap, conn_name, conn_type,
address, add_client_func):
self.queue = Queue(pool_size)
self.queue_build_cap = queue_build_cap
self.conn_name = conn_name
self.conn_type = conn_type
self.address = address
self.add_client_func = add_client_func
self.logger = logging.getLogger(self.__class__.__name__)
def __call__(self):
return _Connection(self.queue, self.conn_name)
def put_client(self, client):
self.queue.put(client)
self.logger.info('Added `%s` client to %s (%s)', self.conn_name,
self.address, self.conn_type)
def build_queue(self):
""" Spawns greenlets to populate the queue and waits up to self.queue_build_cap seconds until the queue is full.
If it never is, raises an exception stating so.
"""
for x in range(self.queue.maxsize):
gevent.spawn(self.add_client_func)
start = datetime.utcnow()
build_until = start + timedelta(seconds=self.queue_build_cap)
while not self.queue.full():
gevent.sleep(0.5)
now = datetime.utcnow()
if now >= build_until:
self.logger.error(
'Built %s/%s %s clients to `%s` within %s seconds, giving up',
self.queue.qsize(), self.queue.maxsize, self.conn_type,
self.address, self.queue_build_cap)
return
self.logger.info(
'%d/%d %s clients connected to `%s` (%s) after %s (cap: %ss)',
self.queue.qsize(), self.queue.maxsize, self.conn_type,
self.address, self.conn_name, now - start,
self.queue_build_cap)
self.logger.info('Obtained %d %s clients to `%s` for `%s`',
self.queue.maxsize, self.conn_type, self.address,
self.conn_name)
class PSPool(object):
LIFE_TIMES = 60 * 1 #sock生命周期
def __init__(self, host, port, max_sock):
self.host = host
self.port = port
self.max_sock = max_sock
self.socks = Queue(maxsize=max_sock)
self.threads = {}
self.sock_times = {}
def init_sock(self):
""" 初始化和pp服务器的socket """
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((self.host, self.port))
return sock
def get(self):
if self.socks.empty():
sock = self.init_sock()
self.sock_times[sock] = time.time()
return sock
return self.socks.get()
def put(self, sock):
times = time.time() - self.sock_times[sock]
if times >= self.LIFE_TIMES or self.socks.full():
self.free(sock)
return
self.socks.put(sock)
def free(self, sock):
self.sock_times.pop(sock, None)
try:
sock.close()
except:
pass
def __enter__(self):
cur_thread = getcurrent()
if cur_thread in self.threads:
raise ValueError('not support reenter')
self.threads[cur_thread] = self.get()
return self.threads[cur_thread]
def __exit__(self, exc_type, exc_val, exc_tb):
sock = self.threads.pop(getcurrent())
if exc_type is None:
self.put(sock)
else:
self.free(sock)
class PSPool(object):
LIFE_TIMES = 60 * 1 #sock生命周期
def __init__(self, host, port, max_sock):
self.host = host
self.port = port
self.max_sock = max_sock
self.socks = Queue(maxsize=max_sock)
self.threads = {}
self.sock_times = {}
def init_sock(self):
""" 初始化和pp服务器的socket """
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((self.host, self.port))
return sock
def get(self):
if self.socks.empty():
sock = self.init_sock()
self.sock_times[sock] = time.time()
return sock
return self.socks.get()
def put(self, sock):
times = time.time() - self.sock_times[sock]
if times >= self.LIFE_TIMES or self.socks.full():
self.free(sock)
return
self.socks.put(sock)
def free(self, sock):
self.sock_times.pop(sock, None)
try:
sock.close()
except:
pass
def __enter__(self):
cur_thread = getcurrent()
if cur_thread in self.threads:
raise ValueError('not support reenter')
self.threads[cur_thread] = self.get()
return self.threads[cur_thread]
def __exit__(self, exc_type, exc_val, exc_tb):
sock = self.threads.pop(getcurrent())
if exc_type is None:
self.put(sock)
else:
self.free(sock)
class ConnectionQueue(object):
""" Holds connections to resources. Each time it's called a connection is fetched from its underlying queue
assuming any connection is still available.
"""
def __init__(self, pool_size, queue_build_cap, conn_name, conn_type, address, add_client_func):
self.queue = Queue(pool_size)
self.queue_build_cap = queue_build_cap
self.conn_name = conn_name
self.conn_type = conn_type
self.address = address
self.add_client_func = add_client_func
self.logger = logging.getLogger(self.__class__.__name__)
def __call__(self):
return _Connection(self.queue, self.conn_name)
def put_client(self, client):
self.queue.put(client)
self.logger.info('Added `%s` client to %s (%s)', self.conn_name, self.address, self.conn_type)
def build_queue(self):
""" Spawns greenlets to populate the queue and waits up to self.queue_build_cap seconds until the queue is full.
If it never is, raises an exception stating so.
"""
for x in range(self.queue.maxsize):
gevent.spawn(self.add_client_func)
start = datetime.utcnow()
build_until = start + timedelta(seconds=self.queue_build_cap)
while not self.queue.full():
gevent.sleep(0.5)
now = datetime.utcnow()
if now >= build_until:
self.logger.error('Built %s/%s %s clients to `%s` within %s seconds, giving up',
self.queue.qsize(), self.queue.maxsize, self.conn_type, self.address, self.queue_build_cap)
return
self.logger.info('%d/%d %s clients connected to `%s` (%s) after %s (cap: %ss)',
self.queue.qsize(), self.queue.maxsize, self.conn_type, self.address, self.conn_name, now - start,
self.queue_build_cap)
self.logger.info('Obtained %d %s clients to `%s` for `%s`', self.queue.maxsize, self.conn_type, self.address, self.conn_name)
class ConnectionQueue(object):
""" Holds connections to resources. Each time it's called a connection is fetched from its underlying queue
assuming any connection is still available.
"""
def __init__(self, pool_size, queue_build_cap, conn_name, conn_type,
address, add_client_func):
self.queue = Queue(pool_size)
self.queue_build_cap = queue_build_cap
self.conn_name = conn_name
self.conn_type = conn_type
self.address = address
self.add_client_func = add_client_func
self.keep_connecting = True
self.lock = RLock()
# How many add_client_func instances are running currently,
# must be updated with self.lock held.
self.in_progress_count = 0
self.logger = logging.getLogger(self.__class__.__name__)
def __call__(self):
return _Connection(self.queue, self.conn_name)
def put_client(self, client):
with self.lock:
if self.queue.full():
is_accepted = False
msg = 'Skipped adding a superfluous `%s` client to %s (%s)'
log_func = self.logger.info
else:
self.queue.put(client)
is_accepted = True
msg = 'Added `%s` client to %s (%s)'
log_func = self.logger.info
log_func(msg, self.conn_name, self.address, self.conn_type)
return is_accepted
def _build_queue(self):
start = datetime.utcnow()
build_until = start + timedelta(seconds=self.queue_build_cap)
suffix = 's ' if self.queue.maxsize > 1 else ' '
try:
while self.keep_connecting and not self.queue.full():
gevent.sleep(5)
now = datetime.utcnow()
self.logger.info(
'%d/%d %s clients obtained to `%s` (%s) after %s (cap: %ss)',
self.queue.qsize(), self.queue.maxsize, self.conn_type,
self.address, self.conn_name, now - start,
self.queue_build_cap)
if now >= build_until:
# Log the fact that the queue is not full yet
self.logger.info(
'Built %s/%s %s clients to `%s` within %s seconds, sleeping until %s (UTC)',
self.queue.qsize(), self.queue.maxsize, self.conn_type,
self.address, self.queue_build_cap,
datetime.utcnow() +
timedelta(seconds=self.queue_build_cap))
# Sleep for a predetermined time
gevent.sleep(self.queue_build_cap)
# Spawn additional greenlets to fill up the queue but make sure not to spawn
# more greenlets than there are slots in the queue still available.
with self.lock:
if self.in_progress_count < self.queue.maxsize:
self._spawn_add_client_func(self.queue.maxsize -
self.in_progress_count)
start = datetime.utcnow()
build_until = start + timedelta(
seconds=self.queue_build_cap)
if self.keep_connecting:
self.logger.info('Obtained %d %s client%sto `%s` for `%s`',
self.queue.maxsize, self.conn_type, suffix,
self.address, self.conn_name)
else:
self.logger.info('Skipped building a queue to `%s` for `%s`',
self.address, self.conn_name)
# Ok, got all the connections
return
except KeyboardInterrupt:
self.keep_connecting = False
def _spawn_add_client_func_no_lock(self, count):
for x in range(count):
gevent.spawn(self.add_client_func)
self.in_progress_count += 1
def _spawn_add_client_func(self, count=1):
""" Spawns as many greenlets to populate the connection queue as there are free slots in the queue available.
"""
with self.lock:
if self.queue.full():
logger.info('Queue already full (c:%d) (%s %s)', count,
self.address, self.conn_name)
return
self._spawn_add_client_func_no_lock(count)
def decr_in_progress_count(self):
with self.lock:
self.in_progress_count -= 1
def build_queue(self):
""" Spawns greenlets to populate the queue and waits up to self.queue_build_cap seconds until the queue is full.
If it never is, raises an exception stating so.
"""
self._spawn_add_client_func(self.queue.maxsize)
# Build the queue in background
gevent.spawn(self._build_queue)
class APIRetreiver(object):
def __init__(self, config, **options):
if not isinstance(config, dict):
raise TypeError("Expected a dict as config, got {}".format(
type(config)))
self.api_host = config.get('api_host')
self.api_version = config.get('api_version')
self.api_key = config.get('api_key')
if 'api_extra_params' in options:
self._extra = options.get('api_extra_params')
self.tender_queue = Queue(maxsize=config.get('queue_max_size', 250))
self.filter_callback = options.get('filter_callback', lambda x: x)
self.forward_worker_dead = Event()
self.forward_worker_dead.set()
self.backward_worker_dead = Event()
self.backward_worker_dead.set()
self._init_clients()
def _init_clients(self):
logger.info('Sync: Init clients')
self.forward_client = APICLient(self.api_key, self.api_host,
self.api_version)
self.backward_client = APICLient(self.api_key, self.api_host,
self.api_version)
self.origin_cookie = self.forward_client.session.cookies
self.backward_client.session.cookies = self.origin_cookie
def _get_sync_point(self):
logger.info('Sync: initializing sync')
forward = {'feed': 'changes'}
backward = {'feed': 'changes', 'descending': '1'}
if getattr(self, '_extra', ''):
[x.update(self._extra) for x in [forward, backward]]
r = self.backward_client.get_tenders(backward)
backward['offset'] = r['next_page']['offset']
forward['offset'] = r['prev_page']['offset']
logger.error(forward)
self.tender_queue.put(filter(self.filter_callback, r['data']))
logger.info('Sync: initial sync params forward: '
'{}, backward: {}'.format(forward, backward))
return forward, backward
def _start_sync_workers(self):
forward, backward = self._get_sync_point()
self.workers = [
gevent.spawn(self._forward_worker, forward),
gevent.spawn(self._backward_worker, backward),
]
def _forward_worker(self, params):
worker = "Forward worker:"
logger.info('{} starting'.format(worker))
r = self.forward_client.get_tenders(params)
if self.forward_client.session.cookies != self.origin_cookie:
raise LBMismatchError
try:
while True:
try:
while r['data']:
try:
self.tender_queue.put(
filter(self.filter_callback, r['data']))
except Full:
while self.tender_queue.full():
gevent.sleep(QUEUE_FULL_DELAY)
self.tender_queue.put(
filter(self.filter_callback, r['data']))
params['offset'] = r['next_page']['offset']
r = self.forward_client.get_tenders(params)
if self.forward_client.session.cookies != self.origin_cookie:
raise LBMismatchError
if r['data']:
gevent.sleep(FORWARD_WORKER_SLEEP)
logger.warn('{} got empty listing. Sleep'.format(worker))
gevent.sleep(ON_EMPTY_DELAY)
except LBMismatchError:
logger.info('LB mismatch error on backward worker')
self.reinit_clients.set()
except Exception as e:
logger.error("{} down! Error: {}".format(worker, e))
self.forward_worker_dead.set()
else:
logger.error("{} finished.".format(worker))
def _backward_worker(self, params):
worker = "Backward worker: "
logger.info('{} staring'.format(worker))
try:
while True:
try:
r = self.backward_client.get_tenders(params)
if not r['data']:
logger.debug(
'{} empty listing..exiting'.format(worker))
break
gevent.sleep(BACKWARD_WOKER_DELAY)
if self.backward_client.session.cookies != self.origin_cookie:
raise LBMismatchError
try:
self.tender_queue.put(
filter(self.filter_callback, r['data']))
except Full:
logger.error('full queue')
while self.tender_queue.full():
gevent.sleep(QUEUE_FULL_DELAY)
self.tender_queue.put(
filter(self.filter_callback, r['data']))
params['offset'] = r['next_page']['offset']
except LBMismatchError:
logger.info('{} LB mismatch error'.format(worker))
if not self.reinit_clients.is_set():
self.reinit_clients.set()
except Exception as e:
logger.error("{} down! Error: {}".format(worker, e))
self.forward_worker_dead.set()
else:
logger.error("{} finished.".format(worker))
def _restart_workers(self):
self._init_clients()
gevent.killall(self.workers)
self._start_sync_workers()
return self.workers
def get_tenders(self):
self._start_sync_workers()
forward, backward = self.workers
try:
while True:
if self.tender_queue.empty():
gevent.sleep(EMPTY_QUEUE_DELAY)
if (forward.dead or forward.ready()) or \
(backward.dead and not backward.successful()):
forward, backward = self._restart_workers()
while not self.tender_queue.empty():
yield self.tender_queue.get()
except Exception as e:
logger.error(e)
class ConnectionQueue(object):
""" Holds connections to resources. Each time it's called a connection is fetched from its underlying queue
assuming any connection is still available.
"""
def __init__(self, pool_size, queue_build_cap, conn_name, conn_type, address, add_client_func):
self.queue = Queue(pool_size)
self.queue_build_cap = queue_build_cap
self.conn_name = conn_name
self.conn_type = conn_type
self.address = address
self.add_client_func = add_client_func
self.keep_connecting = True
self.logger = logging.getLogger(self.__class__.__name__)
def __call__(self):
return _Connection(self.queue, self.conn_name)
def put_client(self, client):
self.queue.put(client)
self.logger.info('Added `%s` client to %s (%s)', self.conn_name, self.address, self.conn_type)
def _build_queue(self):
start = datetime.utcnow()
build_until = start + timedelta(seconds=self.queue_build_cap)
suffix = 's ' if self.queue.maxsize > 1 else ' '
try:
while self.keep_connecting and not self.queue.full():
gevent.sleep(0.5)
now = datetime.utcnow()
self.logger.info('%d/%d %s clients obtained to `%s` (%s) after %s (cap: %ss)',
self.queue.qsize(), self.queue.maxsize,
self.conn_type, self.address, self.conn_name, now - start, self.queue_build_cap)
if now >= build_until:
# Log the fact that the queue is not full yet
self.logger.warn('Built %s/%s %s clients to `%s` within %s seconds, sleeping until %s (UTC)',
self.queue.qsize(), self.queue.maxsize, self.conn_type, self.address, self.queue_build_cap,
datetime.utcnow() + timedelta(seconds=self.queue_build_cap))
# Sleep for a predetermined time
gevent.sleep(self.queue_build_cap)
# Spawn additional greenlets to fill up the queue
self._spawn_add_client_func(self.queue.maxsize - self.queue.qsize())
start = datetime.utcnow()
build_until = start + timedelta(seconds=self.queue_build_cap)
if self.keep_connecting:
self.logger.info('Obtained %d %s client%sto `%s` for `%s`', self.queue.maxsize, self.conn_type, suffix,
self.address, self.conn_name)
else:
self.logger.info('Skipped building a queue to `%s` for `%s`', self.address, self.conn_name)
# Ok, got all the connections
return
except KeyboardInterrupt:
self.keep_connecting = False
def _spawn_add_client_func(self, count):
""" Spawns as many greenlets to populate the connection queue as there are free slots in the queue available.
"""
for x in range(count):
gevent.spawn(self.add_client_func)
def build_queue(self):
""" Spawns greenlets to populate the queue and waits up to self.queue_build_cap seconds until the queue is full.
If it never is, raises an exception stating so.
"""
self._spawn_add_client_func(self.queue.maxsize)
# Build the queue in background
gevent.spawn(self._build_queue)
class GoGame(game_pb2_grpc.GameServicer):
def finishNewThread(self):
self.finish = 0
def __init__(self, maxsize=10):
self.finish = 1
self.connectionTime = 0
self._players = {}
self.gc_pools = Queue(maxsize=maxsize)
self.greenlets = []
self.start_console()
self.lock = False
# def heartBeat():
# while(self.finish==1):
# #lock this block
# #connection is broken
# if (self.connectionTime > 5):
# self.lock = True
# #clear the board now
# for gcItem in self._players.values():
# gcItem.reqChan.put_nowait("clear_board")
# while gcItem.respChan.empty():
# sleep(0)
# gcItem.respChan.get()
# #free all game contexts
# keys = []
# for playerId in self._players.keys():
# keys.append(playerId)
# for playerId in keys:
# self.gc_pools.put_nowait(self._players.pop(playerId))
# print("connection out of time, free all gc")
# self.connectionTime = -1
# self.lock = False
#calculate for the connection time
# self.connectionTime = self.connectionTime + 1
# sleep(1)
# try:
# self.newThread = threading.Thread(target=heartBeat)
# self.newThread.start()
# except:
# print("Error: unable to start thread")
#start the console module
def _start_console(self):
#create a game context first
#the gc is just the game context below
gc = GameContext()
#add the game context to the pool( 10 gc is allowed to add to the pool in default)
self.gc_pools.put_nowait(gc)
#define 2 functions here
#the console of gc is the GoConsoleGTP
def actor(batch):
return gc.console.actor(batch)
#prompt a command of train
#now I consider the batch as a kind of operaion
def train(batch):
gc.console.prompt("DF Train> ", batch)
#the first gc is just the game context below while the second one
#is the game context in source code
gc.console.evaluator.setup(sampler=env["sampler"], mi=mi)
gc.console.GC.reg_callback_if_exists("actor_black", actor)
gc.console.GC.reg_callback_if_exists("human_actor", gc.human_actor)
gc.console.GC.reg_callback_if_exists("train", train)
gc.console.GC.start()
gc.console.GC.GC.getClient().setRequest(
mi["actor"].step, -1, env['game'].options.resign_thres, -1)
gc.console.evaluator.episode_start(0)
while True:
gc.console.GC.run()
if gc.console.exit:
break
gc.console.GC.stop()
def start_console(self):
while not self.gc_pools.full():
g = gevent.spawn(self._start_console)
g.start()
gevent.sleep(0)
self.greenlets.append(g)
def stop_console(self):
gevent.killall(self.greenlets, timeout=10)
#one game context is related to one game
#the function here will return the protobuf
#the function like clear board, play etc., will block here until the response channel is not empty
def NewGC(self, request, context):
# while (self.lock):
# pass
#offered by the client
playerId = request.id
if not playerId in self._players:
if self.gc_pools.empty():
return google_dot_rpc_dot_status__pb2.Status(
code=google_dot_rpc_dot_code__pb2.RESOURCE_EXHAUSTED,
message="no more gc is available")
self._players[playerId] = self.gc_pools.get()
return google_dot_rpc_dot_status__pb2.Status(
code=google_dot_rpc_dot_code__pb2.OK)
def FreeGC(self, request, context):
# while (self.lock):
# pass
playerId = request.id
if playerId not in self._players:
return google_dot_rpc_dot_status__pb2.Status(
code=google_dot_rpc_dot_code__pb2.NOT_FOUND)
self.gc_pools.put_nowait(self._players.pop(playerId))
return google_dot_rpc_dot_status__pb2.Status(
code=google_dot_rpc_dot_code__pb2.OK)
def ClearBoard(self, request, context):
# while (self.lock):
# pass
gc = self._players[request.id]
#now the game will send message to the game context`s channel,
# then those contexts will deal with them orderly
gc.reqChan.put_nowait("clear_board")
while gc.respChan.empty():
gevent.sleep(0)
debug_print("Clear board for player {id}".format(id=request.id))
return gc.respChan.get()
def Play(self, request, context):
# while (self.lock):
# pass
gc = self._players[request.player.id]
try:
gc.action = gc.console.move2action(request.move)
except ValueError as e:
return game_pb2.Reply(status=google_dot_rpc_dot_status__pb2.Status(
code=google_dot_rpc_dot_code__pb2.INVALID_ARGUMENT,
message=str(e)))
gc.reqChan.put_nowait("play")
while gc.respChan.empty():
gevent.sleep(0)
return gc.respChan.get()
def GenMove(self, request, context):
# while (self.lock):
# pass
gc = self._players[request.id]
gc.reqChan.put_nowait("genmove")
#until the gc make response this object will be blocked
while gc.respChan.empty():
gevent.sleep(0)
return gc.respChan.get()
def Pass(self, request, context):
# while (self.lock):
# pass
gc = self._players[request.id]
gc.reqChan.put_nowait("pass")
while gc.respChan.empty():
gevent.sleep(0)
return gc.respChan.get()
def Resign(self, request, context):
# while (self.lock):
# pass
gc = self._players[request.id]
gc.reqChan.put_nowait("resign")
print("put")
while gc.respChan.empty():
gevent.sleep(0)
print("return value")
return gc.respChan.get()
def HeartBeat(self, request, context):
# while (self.lock):
# pass
self.connectionTime = 0
return game_pb2.BeatReply(beatReply=1)
class FrontendHandleBase(object):
""" This base class for frontend handle """
def __init__(self, handle_id, session, plugin, opaque_id=None):
self.handle_id = handle_id
self.opaque_id = opaque_id
self._session = session
self._plugin = plugin
self._has_destroy = False
self.created = time.time()
self.plugin_package_name = plugin.get_package()
self._async_message_queue = Queue(maxsize=1024)
self._async_message_greenlet = gevent.spawn(
self._async_message_handler_routine)
def detach(self):
if self._has_destroy:
return
self._has_destroy = True
# stop async message greenlet
if not self._async_message_queue.full():
self._async_message_queue.put(stop_message)
self._async_message_greenlet = None
# send detach event
event = create_janus_msg('detached', self._session.session_id)
event['sender'] = self.handle_id
if self.opaque_id:
event['opaque_id'] = self.opaque_id
self._session.notify_event(event)
log.info('handle {} is detach from plugin {}'.format(
self.handle_id, self.plugin_package_name))
def has_destroy(self):
return self._has_destroy
def handle_hangup(self):
raise JanusCloudError('hangup not support\'hangup\'',
JANUS_ERROR_MISSING_REQUEST)
def handle_message(self, transaction, body, jsep=None):
raise JanusCloudError('message not support\'message\'',
JANUS_ERROR_PLUGIN_MESSAGE)
def handle_trickle(self, candidate=None, candidates=None):
raise JanusCloudError('hangup not support\'trickle\'',
JANUS_ERROR_MISSING_REQUEST)
def _enqueue_async_message(self, transaction, body, jsep=None):
self._async_message_queue.put_nowait((transaction, body, jsep))
def _async_message_handler_routine(self):
while not self._has_destroy:
msg = self._async_message_queue.get()
if self._has_destroy or msg == stop_message:
return
try:
transaction, body, jsep = msg
self._handle_async_message(transaction, body, jsep)
except Exception:
log.exception(
'Error when handle async message for handle {}'.format(
self.handle_id))
def _handle_async_message(self, transaction, body, jsep):
""" need to override by subclass """
raise JanusCloudError('async message handler not support\'message\'',
JANUS_ERROR_PLUGIN_MESSAGE)
def _push_plugin_event(self, data, jsep=None, transaction=None):
params = dict()
params['plugindata'] = {
'plugin': self.plugin_package_name,
'data': data
}
if jsep:
params['jsep'] = jsep
self._push_event('event', transaction, **params)
def _push_event(self, method, transaction=None, **kwargs):
if self._has_destroy:
return
event = create_janus_msg(method, self._session.session_id, transaction,
**kwargs)
event['sender'] = self.handle_id
if self.opaque_id:
event['opaque_id'] = self.opaque_id
self._session.notify_event(event)
class WFEngineDaemon(GenericDaemon):
def __init__(self, **kwargs):
from gevent.queue import Queue
super(WFEngineDaemon, self).__init__(**kwargs)
self.task_model = get_model('workflow_task')
self.limit = kwargs.get("query_limit", 10)
self.daemon_id = kwargs.get("daemon_id", get_unique_id())
#worker
self.worker_number = kwargs.get("worker", 4)
self.workers = {}
self.queue_limit = kwargs.get("queue_size", 10)
self.queue = Queue(self.queue_limit)
self.status = None
self.is_worker_started = False
self.tid = 0
self.register_request('pause', usage="Pause to load new task.")
self.register_request('resume', usage="Resume to load new task.")
self.register_request('worker', usage="Show worker information.")
self.register_request('reset', inner=False)
self.mainLoopCommands = ["start_workers"]
def get_server_info(self):
info = []
info.append("%s" % __daemon_name__)
info.append("- version: %s" % __daemon_version__)
info.append("- port: %s" % self.port or __daemon_port__)
info.append("- instance: %s" % self.daemon_id)
return info
def get_worker_name(self, i):
a, b = i//6+1, i % 6
name = "%s%d" % (__WORKER_NAMES__[b], a)
while self.workers.has_key(name):
a = a + 1
name = "%s%d" % (__WORKER_NAMES__[b], a)
return name
def loader(self):
import gevent
while True:
if self.status == STOPPED:
break
if self.status == RUNNING:
# load task from database or redis
# load task from database:
self.load_ready_tasks(self.queue)
gevent.sleep(0)
else:
gevent.sleep(0.5)
self.prints(">>> Loader is stopped at %s" % self.gettimestamp())
return
def worker(self, name, index):
import gevent
from random import randint
work_status = RUNNING
self.workers[name] = Worker(name, RUNNING, index)
w = self.workers[name]
while True:
if w.status == PAUSED:
if self.status == RUNNING:
w.status == RUNNING
if w.status == RUNNING:
if not self.queue.empty():
task = self.queue.get()
self.prints('>>> %s got task %s ' % (name, task[0]))
self.async_deliver(task[1])
else:
if self.status == PAUSED:
self.prints(">>> %s is paused at %s" % (name, self.gettimestamp()))
w.status = PAUSED
if w.status == STOPPED:
break
gevent.sleep(0)
del self.workers[name]
self.prints(">>> %s is stopped at %s" % (name, self.gettimestamp()))
self.worker_number = len(self.workers)
if self.worker_number > 1:
self.prints(">>> still have %d workers at server." % self.worker_number)
elif self.worker_number == 1:
self.prints(">>> only one worker at server." )
else:
self.prints(">>> no running worker at server.")
def load_ready_tasks(self, queue):
from uliweb.orm import Begin, Commit, Rollback, Reset
from gevent.queue import Full
import gevent
if self.queue.full():
gevent.sleep(0.3)
return
Begin()
try:
cond = self.task_model.c.state == 2 # READY
cond = (self.task_model.c.async_status == 0 or self.task_model.c.async_status == None) & cond
query = self.task_model.filter(cond)
query = query.order_by(self.task_model.c.async_deliver_date.asc())
query = query.limit(self.limit)
isFull = False
isEmpty = True
for item in query:
self.tid = self.tid + 1
try:
queue.put_nowait([self.tid, item.id])
item.update(async_status=1)
item.save()
isEmpty = False
except Full:
isFull = True
break
Commit()
if isEmpty and self.debug:
self.prints(">>> [DEBUG] No tasks found %s" % self.gettimestamp())
if isFull:
gevent.sleep(0.3)
if isEmpty:
gevent.sleep(0.3)
except:
Rollback()
import traceback
traceback.print_exc()
def startMainLoop1(self):
self.start_workers()
self.prints(">>> MainLoop is stoped at %s" % self.gettimestamp())
def mainLoop(self, cmd=None):
import gevent
if self.mainLoopCommands:
cmd = self.mainLoopCommands.pop()
if hasattr(self, "do_%s" % cmd):
gevent.spawn(getattr(self, "do_%s" % cmd))
#self.prints("mainLoop ..")
def do_start_workers(self):
import gevent
self.prints(">>> Workers are startting....")
self.status = RUNNING
# one loader
greentlets = [gevent.spawn(self.loader)]
# some workers
for i in range(1, self.worker_number+1):
name = self.get_worker_name(i)
greentlets.append(gevent.spawn(self.worker, name, i))
gevent.joinall(greentlets)
def handle_shutdown(self, req):
self.prints(">>> Worker stopping....")
self.status = STOPPED
return super(WFEngineDaemon, self).handle_shutdown(req)
def handle_pause(self, req):
self.status = PAUSED
self.prints(">>> Loader is paused at %s" % self.gettimestamp())
return self.create_response(True, "The loader is pausing.")
def handle_resume(self, req):
self.status = RUNNING
return self.create_response(True, "The loader is resuming.")
def handle_worker(self, req):
subcmd = req.msg
data = req.data
if subcmd == "add":
import gevent
if data and data.isdigit():
addcount = int(data)
else:
addcount = 1
msg = []
for i in range(1, addcount+1):
self.worker_number = self.worker_number + 1
name = self.get_worker_name(self.worker_number)
greenlet = gevent.spawn(self.worker, name, self.worker_number)
msg.append("new worker %s added" % name)
return self.create_response(True, "\n".join(msg))
if subcmd == "show":
if data:
if self.workers.has_key(data):
w = self.workers[name]
msg = "%10s %10s" % (w.index, w.name, w.status)
return self.create_response(True, msg)
else:
return self.create_response(False, "cannot find worker %s" % data)
if subcmd == "del":
if data:
if data == "all":
msg = []
for name in self.workers:
w = self.workers[name]
w.status = STOPPED
msg.append("worker %s was stopped." % name)
return self.create_response(True, "\n".join(msg))
if self.workers.has_key(data):
w = self.workers[data]
w.status = STOPPED
return self.create_response(True, "worker %s was stopped." % data)
return self.create_response(False, "cannot find worker %s" % data)
msg = []
for name in sorted(self.workers):
w = self.workers[name]
msg.append("%10s %10s" % (w.name, w.status))
return self.create_response(True, "\n".join(msg))
def handle_reset(self, req):
Task = self.task_model
count = 0
for item in Task.filter(Task.c.async_status == 1):
item.async_status = None
item.save()
count = count + 1
return self.create_response(True, "reset %d tasks in database." % count)
def async_deliver(self, task_id):
from redbreast.serializable import Workflow, Task
task_obj = self.task_model.get(task_id)
if task_obj.state == 2: #READY
wf_id = task_obj._workflow_
workflow = Workflow.load(wf_id, operator=task_obj._modified_user_)
task = workflow.get_task(task_obj.uuid)
self.prints("------------------------------------------------")
self.prints("spec %s %s(%s)" % (workflow.get_spec_name(), task.get_name(), task.get_spec_name()))
message = task.deliver_msg
next_tasks = task.next_tasks
workflow.deliver(task_obj.uuid, message=message, next_tasks=next_tasks, async=False)
task_obj.update(async_status=0)
task_obj.save()
class BackendHandle(object):
""" This backend handle represents a Janus handle """
def __init__(self, handle_id, plugin_package_name, session, opaque_id=None, handle_listener=None):
self.handle_id = handle_id
self.plugin_package_name = plugin_package_name
self.opaque_id = opaque_id
self._session = session
self._has_detach = False
self._handle_listener = handle_listener
self._async_event_queue = Queue(maxsize=1024)
self._async_event_greenlet = gevent.spawn(self._async_event_handler_routine)
def detach(self):
""" detach this handle from the session
return:
no value
note: no exception would be raised
"""
if self._has_detach:
return
self._has_detach = True
# stop async event greenlet
if not self._async_event_queue.full():
self._async_event_queue.put(stop_message)
self._async_event_greenlet = None
if self._session:
self._session.on_handle_detached(self.handle_id)
try:
detach_message = create_janus_msg('detach', handle_id=self.handle_id)
self._session.send_request(detach_message)
except Exception:
log.exception('Detach backend handle {} error'.format(self.handle_id))
self._session = None
def send_message(self, body, jsep=None):
if self._has_detach:
raise JanusCloudError('backend handle {} has been destroyed'.format(self.handle_id),
JANUS_ERROR_PLUGIN_DETACH)
params = dict()
params['body'] = body
if jsep:
params['jsep'] = jsep
message = create_janus_msg('message', handle_id=self.handle_id, **params)
response = self._session.send_request(message)
if response['janus'] == 'event' or response['janus'] == 'success':
data = response['plugindata']['data']
reply_jsep = response.get('jsep')
return data, reply_jsep
elif response['janus'] == 'error':
raise JanusCloudError(response['error']['reason'], response['error']['code'])
else:
raise JanusCloudError(
'unknown backend response {}'.format(response),
JANUS_ERROR_BAD_GATEWAY)
def send_trickle(self, candidate=None, candidates=None):
if self._has_detach:
raise JanusCloudError('backend handle {} has been destroyed'.format(self.handle_id),
JANUS_ERROR_PLUGIN_DETACH)
if candidate is None and candidates is None:
raise JanusCloudError('Missing mandatory element (candidate|candidates)',
JANUS_ERROR_MISSING_MANDATORY_ELEMENT)
if candidate and candidates:
raise JanusCloudError('Can\'t have both candidate and candidates',
JANUS_ERROR_INVALID_JSON)
params = {}
if candidate:
params['candidate'] = candidate
if candidates:
params['candidates'] = candidates
trickle_msg = create_janus_msg('trickle', handle_id=self.handle_id, **params)
response = self._session.send_request(trickle_msg, ignore_ack=False)
if response['janus'] == 'ack':
pass # successful
elif response['janus'] == 'error':
raise JanusCloudError(response['error']['reason'], response['error']['code'])
else:
raise JanusCloudError(
'unknown backend response {}'.format(response),
JANUS_ERROR_BAD_GATEWAY)
def send_hangup(self):
if self._has_detach:
raise JanusCloudError('backend handle {} has been destroyed'.format(self.handle_id),
JANUS_ERROR_PLUGIN_DETACH)
hangup_msg = create_janus_msg('hangup', handle_id=self.handle_id)
response = self._session.send_request(hangup_msg)
if response['janus'] == 'success':
pass # successful
elif response['janus'] == 'error':
raise JanusCloudError(response['error']['reason'], response['error']['code'])
else:
raise JanusCloudError(
'unknown backend response {}'.format(response),
JANUS_ERROR_BAD_GATEWAY)
def on_async_event(self, event_msg):
if not self._async_event_queue.full():
self._async_event_queue.put(event_msg)
else:
# drop the event
log.error("backend handle {} async event queue is full, drop the receiving event".format(self.handle_id))
def on_close(self):
if self._has_detach:
return
self._has_detach = True
# stop async event greenlet
if not self._async_event_queue.full():
self._async_event_queue.put(stop_message)
self._async_event_greenlet = None
self._session = None
if self._handle_listener:
try:
self._handle_listener.on_close(self.handle_id)
except Exception:
log.exception('on_close() exception for backend handle {}'.format(self.handle_id))
def _async_event_handler_routine(self):
while not self._has_detach:
event_msg = self._async_event_queue.get()
if self._has_detach or event_msg == stop_message:
return
try:
if self._handle_listener:
self._handle_listener.on_async_event(event_msg)
except Exception:
log.exception('Error when handle async event for backend handle {}'.format(self.handle_id))
class DriverPool(object):
""" Create a pool of available Selenium containers for processing.
Args:
size (int): maximum concurrent tasks. Must be at least ``2``.
driver_cls (WebDriver):
driver_cls_args (tuple):
driver_cls_kw (dict):
use_proxy (bool):
factory (:obj:`~selenium_docker.base.ContainerFactory`):
name (str):
logger (:obj:`logging.Logger`):
Example::
pool = DriverPool(size=2)
urls = [
'https://google.com',
'https://reddit.com',
'https://yahoo.com',
'http://ksl.com',
'http://cnn.com'
]
def get_title(driver, url):
driver.get(url)
return driver.title
for result in pool.execute(get_title, urls):
print(result)
"""
INNER_THREAD_SLEEP = 0.5
"""float: essentially our polling interval between tasks and checking
when tasks have completed.
"""
PROXY_CLS = SquidProxy
""":obj:`~selenium_docker.proxy.AbstractProxy`: created for the pool
when ``use_proxy=True`` during pool instantiation.
"""
def __init__(self, size, driver_cls=ChromeDriver, driver_cls_args=None,
driver_cls_kw=None, use_proxy=True, factory=None, name=None,
logger=None):
self.size = max(2, size)
self.name = name or gen_uuid(6)
self.factory = factory or ContainerFactory.get_default_factory()
self.logger = logger or getLogger(
'%s.DriverPool.%s' % (__name__, self.name))
self._driver_cls = driver_cls
self._driver_cls_args = driver_cls_args or tuple()
self._driver_cls_kw = driver_cls_kw or dict()
self._drivers = Queue(maxsize=self.size)
# post init inspections
if not hasattr(self._driver_cls, 'CONTAINER'):
raise DriverPoolValueError('driver_cls must extend DockerDriver')
if not isiterable(self._driver_cls_args):
raise DriverPoolValueError(
'%s is not iterable' % self._driver_cls_args)
if not isinstance(self._driver_cls_kw, Mapping):
raise DriverPoolValueError(
'%s is not a valid mapping' % self._driver_cls_kw)
# determine proxy usage
self.proxy = None
self._use_proxy = use_proxy # type: bool
# deferred instantiation
self._pool = None # type: Pool
self._results = None # type: Queue
self._tasks = None # type: JoinableQueue
self._processing = False # type: bool
self.__feeder_green = None # type: gevent.Greenlet
def __repr__(self):
return '<DriverPool-%s(size=%d,driver=%s,proxy=%s,async=%s)>' % (
self.name, self.size, self._driver_cls.BROWSER,
self._use_proxy, self.is_async)
def __iter__(self):
return self.results(block=self.is_async)
def __del__(self):
try:
self.close()
except Exception as e:
if hasattr(self, 'logger'):
self.logger.exection(e, exc_info=False)
@property
def is_processing(self):
"""bool: whether or not we're currently processing tasks. """
return self._processing
@property
def is_async(self):
"""bool: returns True when asynchronous processing is happening. """
return self.__feeder_green is not None
def __bootstrap(self):
""" Prepare this driver pool instance to batch execute task items. """
if self.is_processing:
# cannot run two executions simultaneously
raise DriverPoolRuntimeException(
'cannot bootstrap pool, already running')
if self._results and self._results.qsize(): # pragma: no cover
self.logger.debug('pending results being discarded')
if self._tasks and self._tasks.qsize(): # pragma: no cover
self.logger.debug('pending tasks being discarded')
if self._pool: # pragma: no cover
self.logger.debug('killing processing pool')
self._pool.join(timeout=10.0)
self._pool.kill()
self._pool = None
if self._use_proxy and not self.proxy:
# defer proxy instantiation -- since spinning up a squid proxy
# docker container is surprisingly time consuming.
self.logger.debug('bootstrapping squid proxy')
self.proxy = self.PROXY_CLS(factory=self.factory)
self.logger.debug('bootstrapping pool processing')
self._processing = True
self._results = Queue()
self._tasks = JoinableQueue()
self._load_drivers()
# create our processing pool with headroom over the number of drivers
# requested for this processing pool.
self._pool = Pool(size=self.size + math.ceil(self.size * 0.25))
def __cleanup(self, force=False):
""" Stop and remove the web drivers and their containers. This function
should not remove pending tasks or results. It should be possible to
cleanup all the external resources of a driver pool and still extract
the results of the work that was completed.
Raises:
DriverPoolRuntimeException: when attempting to cleanup an
environment while processing is still happening, and forcing
the cleanup is set to ``False``.
SeleniumDockerException: when a driver instance or container
cannot be closed properly.
Returns:
None
"""
if self.is_processing and not force: # pragma: no cover
raise DriverPoolRuntimeException(
'cannot cleanup driver pool while executing')
self._processing = False
squid = None # type: gevent.Greenlet
error = None # type: SeleniumDockerException
if self.proxy:
self.logger.debug('closing squid proxy')
squid = gevent.spawn(self.proxy.quit)
if self._pool: # pragma: no cover
self.logger.debug('emptying task pool')
if not force:
self._pool.join(timeout=10.0)
self._pool.kill(block=False,
timeout=10.0)
self._pool = None
self.logger.debug('closing all driver containers')
while not self._drivers.empty():
d = self._drivers.get(block=True)
try:
d.quit()
except SeleniumDockerException as e: # pragma: no cover
self.logger.exception(e, exc_info=True)
if not force:
error = e
if self.proxy:
squid.join()
self.proxy = None
if error: # pragma: no cover
raise error
def _load_driver(self, and_add=True):
""" Load a single web driver instance and container. """
args = self._driver_cls_args
kw = dict(self._driver_cls_kw)
kw.update({
'proxy': self.proxy,
'factory': self.factory,
})
driver = self._driver_cls(*args, **kw)
if and_add:
self._drivers.put(driver)
return driver
def _load_drivers(self):
""" Load the web driver instances and containers.
Raises:
DriverPoolRuntimeException: when the requested number of drivers
for the given pool size cannot be created for some reason.
Returns:
None
"""
if not self._drivers.empty(): # pragma: no cover
return
threads = []
for o in range(self.size):
self.logger.debug('creating driver %d of %d', o + 1, self.size)
thread = gevent.spawn(self._load_driver)
threads.append(thread)
for t in reversed(threads):
t.join()
if not self._drivers.full():
raise DriverPoolRuntimeException(
'unable to fulfill required concurrent drivers, %d of %d' % (
self._drivers.qsize(), self.size))
def _recycle_driver(self, driver):
if not driver:
return
try:
driver.quit()
except Exception as e:
self.logger.exception(e, exc_info=True)
# do NOT add the new driver container to the drivers queue,
# instead this will be handled in the recycle logic that requested
# the driver in the first place. Instead of returning the one it
# received this "new" instance will be put in its placed.
print('RECYCLED!!!!!!')
return self._load_driver(and_add=False)
def add_async(self, *items):
""" Add additional items to the asynchronous processing queue.
Args:
items (list(Any)): list of items that need processing. Each item is
applied one at a time to an available driver from the pool.
Raises:
StopIteration: when all items have been added.
"""
if len(items) == 1 and isinstance(items[0], list):
items = iter(items[0])
if not items:
raise DriverPoolValueError(
'cannot add items with value: %s' % str(items))
item_count = count(items)
self.logger.debug('adding %d additional items to tasks', item_count)
for o in items:
self._tasks.put(o)
def close(self):
""" Force close all the drivers and cleanup their containers.
Returns:
None
"""
self.__cleanup(force=True)
def execute(self, fn, items, preserve_order=False, auto_clean=True,
no_wait=False):
""" Execute a fixed function, blocking for results.
Args:
fn (Callable): function that takes two parameters, ``driver`` and
``task``.
items (list(Any)): list of items that need processing. Each item is
applied one at a time to an available driver from the pool.
preserve_order (bool): should the results be returned in the order
they were supplied via ``items``. It's more performant to
allow results to return in any order.
auto_clean (bool): cleanup docker containers after executing. If
multiple processing tasks are going to be used, it's more
performant to leave the containers running and reuse them.
no_wait (bool): forgo a small sleep interval between finishing
a task and putting the driver back in the available drivers
pool.
Yields:
results: the result for each item as they're finished.
"""
def worker(o):
job_num, item = o
self.logger.debug('doing work on item %d' % job_num)
driver = self._drivers.get(block=True)
ret_val = fn(driver, item)
if not no_wait:
gevent.sleep(self.INNER_THREAD_SLEEP)
self._drivers.put(driver)
return ret_val
if self.__feeder_green:
raise DriverPoolRuntimeException(
'cannot perform a blocking execute while async processing')
self.__bootstrap()
self.logger.debug('starting sync processing')
if preserve_order:
ittr = self._pool.imap
else:
ittr = self._pool.imap_unordered
self.logger.debug('yielding processed results')
for o in ittr(worker, enumerate(items)):
self._results.put(o)
self._results.put(StopIteration)
self.logger.debug('stopping sync processing')
if auto_clean:
self.logger.debug('auto cleanup pool environment')
self.__cleanup(force=True)
return self.results(block=False)
def execute_async(self, fn, items=None, callback=None,
catch=(WebDriverException,), requeue_task=False):
""" Execute a fixed function in the background, streaming results.
Args:
fn (Callable): function that takes two parameters, ``driver`` and
``task``.
items (list(Any)): list of items that need processing. Each item is
applied one at a time to an available driver from the pool.
callback (Callable): function that takes a single parameter, the
return value of ``fn`` when its finished processing and has
returned the driver to the queue.
catch (tuple[Exception]): tuple of Exception classes to catch
during task execution. If one of these Exception classes
is caught during ``fn`` execution the driver that crashed will
attempt to be recycled.
requeue_task (bool): in the event of an Exception being caught
should the task/item that was being worked on be re-added to
the queue of items being processed.
Raises:
DriverPoolValueError: if ``callback`` is not ``None``
or ``callable``.
Returns:
None
"""
def worker(fn, task):
ret_val = None
async_task_id = gen_uuid(12)
self.logger.debug('starting async task %s', async_task_id)
driver = self._drivers.get(block=True)
if isinstance(driver, Exception):
raise driver
try:
ret_val = fn(driver, task)
except catch as e:
self.logger.exception('hihi')
if self.is_processing:
driver = self._recycle_driver(driver)
if requeue_task:
self._tasks.put(task)
finally:
self._results.put(ret_val)
self._drivers.put(driver)
gevent.sleep(self.INNER_THREAD_SLEEP)
return ret_val
def feeder():
self.logger.debug('starting async feeder thread')
while True:
while not self._tasks.empty():
task = self._tasks.get()
if self._pool is None:
break
self._pool.apply_async(
worker,
args=(fn, task,),
callback=greenlet_callback)
gevent.sleep(self.INNER_THREAD_SLEEP)
if self._pool is None and not self.is_processing:
break
return
if callback is None:
def logger(value):
self.logger.debug('%s', value)
callback = logger
def real_callback(cb, value):
if isinstance(value, gevent.GreenletExit):
raise value
else:
cb(value)
greenlet_callback = partial(real_callback, callback)
for f in [fn, callback]:
if not callable(f):
raise DriverPoolValueError(
'cannot use %s, is not callable' % callback)
self.logger.debug('starting async processing')
self.__bootstrap()
if not self.__feeder_green:
self.__feeder_green = gevent.spawn(feeder)
if items:
self.add_async(*items)
def quit(self):
""" Alias for :func:`~DriverPool.close()`. Included for consistency
with driver instances that generally call ``quit`` when they're no
longer needed.
Returns:
None
"""
if self.__feeder_green:
return self.stop_async()
return self.close()
def results(self, block=True):
""" Iterate over available results from processed tasks.
Args:
block (bool): when ``True``, block this call until all tasks have
been processed and all results have been returned. Otherwise
this will continue indefinitely while tasks are dynamically
added to the async processing queue.
Yields:
results: one result at a time as they're finished.
Raises:
StopIteration: when the processing is finished.
"""
est_size = self._results.qsize()
self.logger.debug('there are an estimated %d results', est_size)
if block:
self.logger.debug('blocking for results to finish processing')
while self.is_processing:
while not self._results.empty():
yield self._results.get()
gevent.sleep(self.INNER_THREAD_SLEEP)
if self._tasks.empty() and self._results.empty():
break
raise StopIteration
else:
if est_size > 0:
self.logger.debug('returning as many results as have finished')
self._results.put(StopIteration)
for result in self._results:
yield result
def stop_async(self, timeout=None, auto_clean=True):
""" Stop all the async worker processing from executing.
Args:
timeout (float): number of seconds to wait for pool to finish
processing before killing and closing out the execution.
auto_clean (bool): cleanup docker containers after executing. If
multiple processing tasks are going to be used, it's more
performant to leave the containers running and reuse them.
Returns:
None
"""
self.logger.debug('stopping async processing')
if self.__feeder_green:
self.logger.debug('killing async feeder thread')
gevent.kill(self.__feeder_green)
self.__feeder_green = None
if self._pool:
self.logger.debug('joining async pool before kill')
self._pool.join(timeout=timeout or 1.0)
self._pool.kill(block=False)
tasks_count = self._tasks.qsize()
self.logger.info('%d tasks remained unprocessed', tasks_count)
if auto_clean:
self.logger.debug('auto cleanup pool environment')
self.__cleanup(force=True)
class WFEngineDaemon(GenericDaemon):
def __init__(self, **kwargs):
from gevent.queue import Queue
super(WFEngineDaemon, self).__init__(**kwargs)
self.task_model = get_model('workflow_task')
self.limit = kwargs.get("query_limit", 10)
self.daemon_id = kwargs.get("daemon_id", get_unique_id())
#worker
self.worker_number = kwargs.get("worker", 4)
self.workers = {}
self.queue_limit = kwargs.get("queue_size", 10)
self.queue = Queue(self.queue_limit)
self.status = None
self.is_worker_started = False
self.tid = 0
self.register_request('pause', usage="Pause to load new task.")
self.register_request('resume', usage="Resume to load new task.")
self.register_request('worker', usage="Show worker information.")
self.register_request('reset', inner=False)
self.mainLoopCommands = ["start_workers"]
def get_server_info(self):
info = []
info.append("%s" % __daemon_name__)
info.append("- version: %s" % __daemon_version__)
info.append("- port: %s" % self.port or __daemon_port__)
info.append("- instance: %s" % self.daemon_id)
return info
def get_worker_name(self, i):
a, b = i // 6 + 1, i % 6
name = "%s%d" % (__WORKER_NAMES__[b], a)
while self.workers.has_key(name):
a = a + 1
name = "%s%d" % (__WORKER_NAMES__[b], a)
return name
def loader(self):
import gevent
while True:
if self.status == STOPPED:
break
if self.status == RUNNING:
# load task from database or redis
# load task from database:
self.load_ready_tasks(self.queue)
gevent.sleep(0)
else:
gevent.sleep(0.5)
self.prints(">>> Loader is stopped at %s" % self.gettimestamp())
return
def worker(self, name, index):
import gevent
from random import randint
work_status = RUNNING
self.workers[name] = Worker(name, RUNNING, index)
w = self.workers[name]
while True:
if w.status == PAUSED:
if self.status == RUNNING:
w.status == RUNNING
if w.status == RUNNING:
if not self.queue.empty():
task = self.queue.get()
self.prints('>>> %s got task %s ' % (name, task[0]))
self.async_deliver(task[1])
else:
if self.status == PAUSED:
self.prints(">>> %s is paused at %s" %
(name, self.gettimestamp()))
w.status = PAUSED
if w.status == STOPPED:
break
gevent.sleep(0)
del self.workers[name]
self.prints(">>> %s is stopped at %s" % (name, self.gettimestamp()))
self.worker_number = len(self.workers)
if self.worker_number > 1:
self.prints(">>> still have %d workers at server." %
self.worker_number)
elif self.worker_number == 1:
self.prints(">>> only one worker at server.")
else:
self.prints(">>> no running worker at server.")
def load_ready_tasks(self, queue):
from uliweb.orm import Begin, Commit, Rollback, Reset
from gevent.queue import Full
import gevent
if self.queue.full():
gevent.sleep(0.3)
return
Begin()
try:
cond = self.task_model.c.state == 2 # READY
cond = (self.task_model.c.async_status == 0
or self.task_model.c.async_status == None) & cond
query = self.task_model.filter(cond)
query = query.order_by(self.task_model.c.async_deliver_date.asc())
query = query.limit(self.limit)
isFull = False
isEmpty = True
for item in query:
self.tid = self.tid + 1
try:
queue.put_nowait([self.tid, item.id])
item.update(async_status=1)
item.save()
isEmpty = False
except Full:
isFull = True
break
Commit()
if isEmpty and self.debug:
self.prints(">>> [DEBUG] No tasks found %s" %
self.gettimestamp())
if isFull:
gevent.sleep(0.3)
if isEmpty:
gevent.sleep(0.3)
except:
Rollback()
import traceback
traceback.print_exc()
def startMainLoop1(self):
self.start_workers()
self.prints(">>> MainLoop is stoped at %s" % self.gettimestamp())
def mainLoop(self, cmd=None):
import gevent
if self.mainLoopCommands:
cmd = self.mainLoopCommands.pop()
if hasattr(self, "do_%s" % cmd):
gevent.spawn(getattr(self, "do_%s" % cmd))
#self.prints("mainLoop ..")
def do_start_workers(self):
import gevent
self.prints(">>> Workers are startting....")
self.status = RUNNING
# one loader
greentlets = [gevent.spawn(self.loader)]
# some workers
for i in range(1, self.worker_number + 1):
name = self.get_worker_name(i)
greentlets.append(gevent.spawn(self.worker, name, i))
gevent.joinall(greentlets)
def handle_shutdown(self, req):
self.prints(">>> Worker stopping....")
self.status = STOPPED
return super(WFEngineDaemon, self).handle_shutdown(req)
def handle_pause(self, req):
self.status = PAUSED
self.prints(">>> Loader is paused at %s" % self.gettimestamp())
return self.create_response(True, "The loader is pausing.")
def handle_resume(self, req):
self.status = RUNNING
return self.create_response(True, "The loader is resuming.")
def handle_worker(self, req):
subcmd = req.msg
data = req.data
if subcmd == "add":
import gevent
if data and data.isdigit():
addcount = int(data)
else:
addcount = 1
msg = []
for i in range(1, addcount + 1):
self.worker_number = self.worker_number + 1
name = self.get_worker_name(self.worker_number)
greenlet = gevent.spawn(self.worker, name, self.worker_number)
msg.append("new worker %s added" % name)
return self.create_response(True, "\n".join(msg))
if subcmd == "show":
if data:
if self.workers.has_key(data):
w = self.workers[name]
msg = "%10s %10s" % (w.index, w.name, w.status)
return self.create_response(True, msg)
else:
return self.create_response(False,
"cannot find worker %s" % data)
if subcmd == "del":
if data:
if data == "all":
msg = []
for name in self.workers:
w = self.workers[name]
w.status = STOPPED
msg.append("worker %s was stopped." % name)
return self.create_response(True, "\n".join(msg))
if self.workers.has_key(data):
w = self.workers[data]
w.status = STOPPED
return self.create_response(
True, "worker %s was stopped." % data)
return self.create_response(False, "cannot find worker %s" % data)
msg = []
for name in sorted(self.workers):
w = self.workers[name]
msg.append("%10s %10s" % (w.name, w.status))
return self.create_response(True, "\n".join(msg))
def handle_reset(self, req):
Task = self.task_model
count = 0
for item in Task.filter(Task.c.async_status == 1):
item.async_status = None
item.save()
count = count + 1
return self.create_response(True,
"reset %d tasks in database." % count)
def async_deliver(self, task_id):
from redbreast.serializable import Workflow, Task
task_obj = self.task_model.get(task_id)
if task_obj.state == 2: #READY
wf_id = task_obj._workflow_
workflow = Workflow.load(wf_id, operator=task_obj._modified_user_)
task = workflow.get_task(task_obj.uuid)
self.prints("------------------------------------------------")
self.prints("spec %s %s(%s)" %
(workflow.get_spec_name(), task.get_name(),
task.get_spec_name()))
message = task.deliver_msg
next_tasks = task.next_tasks
workflow.deliver(task_obj.uuid,
message=message,
next_tasks=next_tasks,
async=False)
task_obj.update(async_status=0)
task_obj.save()
class MsgpackEndpoint(object):
def __init__(self, mode, conn, conn_error_handle, router=None, timeout=5.0, poolsize=10, chunksize=1024*32, pack_encoding='utf-8', unpack_encoding='utf-8'):
self._router = router
self._conn_error_handle = conn_error_handle
self._mode = mode
self._timeout = timeout
self._poolsize = poolsize
self._sendqueue = Queue(maxsize=poolsize)
self._msgpool = dict()
self._pack_encoding = pack_encoding
self._unpack_encoding = unpack_encoding
self._packer = msgpack.Packer(use_bin_type=True, encoding=pack_encoding.encode("utf-8"))
self._unpacker = msgpack.Unpacker(encoding=unpack_encoding.encode("utf-8"))
self._conn = conn
self._chunksize = chunksize
self._connecting = False
self._msgid = 0
self._reading_worker = gevent.spawn(self._reading)
self._sending_worker = gevent.spawn(self._sending)
self._reading_worker.start()
self._sending_worker.start()
def _reading(self):
while True:
if not self._conn:
gevent.sleep(1.0)
continue
try:
data = self._conn.recv(self._chunksize)
except socket.timeout as t:
continue
if not data:
logging.warning("connection closed")
tmpconn = self._conn
self._conn = None
self._conn_error_handle.on_conn_error(tmpconn, self, Exception("connection closed"))
continue
self._unpacker.feed(data)
while True:
try:
msg = self._unpacker.next()
except StopIteration as e:
break
self._parse_msg(msg)
def _sending(self):
while True:
if not self._conn:
gevent.sleep(1.0)
continue
body = self._sendqueue.get()
try:
self._conn.sendall(body)
except Exception as e:
logging.warning("RPCClient._sending:error occured. {}".format(e))
tmpconn = self._conn
self._conn = None
if not self._sendqueue.full():
self._sendqueue.put(body)
self._conn_error_handle.on_conn_error(tmpconn, self, e)
continue
def _parse_msg(self, msg):
if (type(msg) != list and type(msg) != tuple) or len(msg) < 3:
logging.warn("invalid msgpack-rpc msg. type={}, msg={}".format(type(msg), msg))
tmpconn = self._conn
self._conn = None
self._conn_error_handle.on_conn_error(tmpconn, self, Exception("invalid msgpack-rpc msg"))
return
if msg[0] == MSGPACKRPC_RSP and len(msg) == 4 and self._mode & MODECLIENT:
(_, msgid, error, result) = msg
if msgid not in self._msgpool:
logging.warn("unexpected msgid. msgid = {}".format(msgid))
return
msgsit = self._msgpool[msgid]
del self._msgpool[msgid]
msgsit[1] = error
msgsit[2] = result
msgsit[0].set()
elif msg[0] == MSGPACKRPC_REQ and len(msg) == 4 and self._mode & MODESERVER:
(_, msgid, method, params) = msg
func = self._router.get_call(method)
result = None
if not func:
rsp = (MSGPACKRPC_RSP, msgid, "Method not found: {}".format(method), None)
self._sendqueue.put(self._packer.pack(rsp))
return
if not hasattr(func, '__call__'):
rsp = (MSGPACKRPC_RSP, msgid, "Method is not callable: {}".format(method), None)
self._sendqueue.put(self._packer.pack(rsp))
return
try:
result = func(*params)
except Exception as e:
rsp = (MSGPACKRPC_RSP, msgid, "{}".format(e), None)
self._sendqueue.put(self._packer.pack(rsp))
return
rsp = (MSGPACKRPC_RSP, msgid, None, result)
self._sendqueue.put(self._packer.pack(rsp))
elif msg[0] == MSGPACKRPC_NOTIFY and len(msg) == 3 and self._mode & MODESERVER:
(_, method, params) = msg
func = self._router.get_notify(method)
if not func:
logging.warn("Method not found: {}".format(method))
return
if not hasattr(func, '__call__'):
logging.warn("Method is not callable: {}".format(method))
return
try:
func(*params)
except Exception as e:
logging.warn("Exception: {} in notify {}".format(e, method))
return
else:
logging.warn("invalid msgpack-rpc msg {}".format(msg))
tmpconn = self._conn
self._conn = None
self._conn_error_handle.on_conn_error(tmpconn, self, Exception("invalid msgpack-rpc msg"))
return
def attach_conn(self, conn):
if self._conn:
return False
self._conn = conn
return True
def call(self, method, *args):
if not self._conn:
logging.warn("rpc connection closed")
if type(method) != str or type(args) != tuple:
raise Exception("invalid msgpack-rpc request, type(method)={}, type(args)={}".format(type(method), type(args)))
self._msgid += 1
msgid = self._msgid
req = (MSGPACKRPC_REQ, msgid, method, args)
body = self._packer.pack(req)
msgsit = [Event(), None, None]
self._msgpool[msgid] = msgsit
self._sendqueue.put(body)
r = msgsit[0].wait(timeout=self._timeout)
if not r:
raise Exception("msgpack-rpc call timeout after {} seconds, msgid = {}".format(self._timeout, msgid))
if msgsit[1]:
raise Exception("msgpack-rpc call rsp error. {}".format(msgsit[1]))
return msgsit[2]
def notify(self, method, *args):
if not self._conn:
logging.warn("rpc connection closed")
if type(method) != str or type(args) != tuple:
raise Exception("invalid msgpack-rpc request, type(method)={}, type(args)={}".format(type(method), type(args)))
notify = (MSGPACKRPC_NOTIFY, method, args)
self._sendqueue.put(self._packer.pack(notify))
def close(self):
if self._reading_worker:
self._reading_worker.kill()
self._reading_worker = None
if self._sending_worker:
self._sending_worker.kill()
self._sending_worker = None
if self._conn:
self._conn = None
