def stop(self):
log.info("Stopping nodes")
stop_group = Group()
for runner in self._node_runners:
stop_group.spawn(runner.stop)
stop_group.join(raise_error=True)
log.info("Nodes stopped")
class WSGIServer(_WSGIServer):
handler_class = WebSocketWSGIHandler
def __init__(self, *args, **kwargs):
"""
WSGI server that simply tracks websockets
and send them a proper closing handshake
when the server terminates.
Other than that, the server is the same
as its :class:`gevent.pywsgi.WSGIServer`
base.
"""
_WSGIServer.__init__(self, *args, **kwargs)
self._websockets = Group()
def link_websocket_to_server(self, websocket):
logger.info("Managing websocket %s" % format_addresses(websocket))
self._websockets.spawn(websocket.run)
def stop(self, *args, **kwargs):
logger.info("Terminating server and all connected websockets")
for greenlet in self._websockets:
try:
websocket = greenlet._run.im_self
if websocket:
websocket.close(1001, 'Server is shutting down')
except:
pass
_WSGIServer.stop(self, *args, **kwargs)
def run(self):
import gevent.queue
from gevent.pool import Group
gq = gevent.queue.Queue()
group = Group()
group.spawn(transfer_queue, self.q, gq)
run_from_queue(group, gq)
class SlaveRunner(Runner):
def __init__(self, host, port):
super().__init__(host, port)
self.slave_id = socket.gethostname()
self.client = rpc.Client(self.host, self.port)
self.greenlet = Group()
self.greenlet.spawn(self.work)
self.client.send(rpc.Message('slave_ready', None, self.slave_id))
def stats_reporter(self):
while True:
logger.info("starting....." + str(time.time()))
# work = Worker(self.client,self.greenlet)
time.sleep(SLAVE_REPORT_INTERVAL)
self.client.send(
Message('slave_complete', None, socket.gethostname()))
def work(self):
while True:
print(">>>>>>>>>><<<<<<<<")
msg = self.client.recv()
logger.info(msg.type)
if msg.type == 'slave_start':
pass
elif msg.type == 'slave_stop':
# self.stop()
self.client.send(Message("slave_stop", None, self.slave_id))
self.client.send(Message("slave_ready", None, self.slave_id))
elif msg.type == 'slave_quit':
logger.info("Got quit message from master, shutting down...")
self.stop()
self.greenlet.kill()
def pipeline(stages, initial_data):
monitors = Group()
# Make sure items in initial_data are iterable.
if not isinstance(initial_data, types.GeneratorType):
try:
iter(initial_data)
except:
raise TypeError('initial_data must be iterable')
# The StopIteration will bubble through the queues as it is reached.
# Once a stage monitor sees it, it indicates that the stage will read
# no more data and the monitor can wait for the current work to complete
# and clean up.
if hasattr(initial_data, 'append'):
initial_data.append(StopIteration)
if not stages:
return PipelineResult(monitors, [])
# chain stage queue io
# Each stage shares an output queue with the next stage's input.
qs = [initial_data] + [Queue() for _ in range(len(stages))]
for stage, in_q, out_q in zip(stages, qs[:-1], qs[1:]):
stage.in_q = in_q
stage.out_q = out_q
monitors.spawn(stage_monitor, stage)
gevent.sleep(0)
return PipelineResult(monitors, stages[-1].out_q)
def test_quiet_group():
from gevent.pool import Group
group = Group()
group.spawn(divide_by_zero)
group.spawn(divide_by_zero)
with raises_from(ZeroDivisionError, 'divide_by_zero'):
group.join(raise_error=True)
class WSGIServer(_WSGIServer):
handler_class = WebSocketWSGIHandler
def __init__(self, *args, **kwargs):
"""
WSGI server that simply tracks websockets
and send them a proper closing handshake
when the server terminates.
Other than that, the server is the same
as its :class:`gevent.pywsgi.WSGIServer`
base.
"""
_WSGIServer.__init__(self, *args, **kwargs)
self._websockets = Group()
def link_websocket_to_server(self, websocket):
logger.info("Managing websocket %s" % format_addresses(websocket))
self._websockets.spawn(websocket.run)
def stop(self, *args, **kwargs):
logger.info("Terminating server and all connected websockets")
for greenlet in self._websockets:
try:
websocket = greenlet._run.im_self
if websocket:
websocket.close(1001, 'Server is shutting down')
except:
pass
_WSGIServer.stop(self, *args, **kwargs)
class MbtClient(aWebSocketClient):
def __init__(self, *args):
super().__init__(*args)
self.callback_dict = {}
self.greenlets = Group()
self.greenlets.spawn(self.onMessage)
def onMessage(self):
while True:
try:
#收到的data类型为ws4py.messaging.TextMessage
data = self.receive()
#data.data是bytes类型，先转化为str
parsed_data = json.loads(data.data.decode())
cmd = parsed_data["head"]["cmd"]
cmd_name = CommandMapping(cmd)
body_dict = json.loads(parsed_data["body"])
parsed_data["body"] = body_dict
self.callback_dict[cmd_name] = DotDict(parsed_data)
logger.info("Receive data from {}:{}".format(cmd_name, parsed_data))
except ValueError:
logger.warn("Receive unknown commandID: {}".format(cmd))
except TypeError:
logger.error("Receive invalid data that is not JSON serializable:{}".format(data.data))
def closed(self, code, reason):
self.greenlets.kill()
logger.debug("Socket was closed:code-{};reason-{}".format(code, reason))
def test_parallel_folder_syncs(db, folder_name_mapping, default_account,
monkeypatch):
# test that when we run save_folder_names in parallel, we only create one
# tag for that folder. this happens when the CondstoreFolderSyncEngine
# checks for UID changes.
# patching the heartbeat clear means that we force the first greenlet to
# wait around (there is a deleted folder in folder_name_mapping), thereby
# assuring that the second greenlet will overtake it and force any
# potential race condition around tag creation.
def clear_heartbeat_patch(w, x, y, z):
gevent.sleep(1)
monkeypatch.setattr('inbox.heartbeat.store.HeartbeatStore.remove_folders',
clear_heartbeat_patch)
log = get_logger()
group = Group()
with mailsync_session_scope() as db_session:
group.spawn(save_folder_names, log, default_account.id,
folder_name_mapping, db_session)
with mailsync_session_scope() as db_session:
group.spawn(save_folder_names, log, default_account.id,
folder_name_mapping, db_session)
group.join()
with mailsync_session_scope() as db_session:
account = db_session.query(Account).get(default_account.id)
random_tags = db_session.query(Tag).filter_by(
namespace_id=account.namespace.id, name='random')
assert random_tags.count() == 1
def test_parallel_folder_syncs(db, folder_name_mapping, monkeypatch):
# test that when we run save_folder_names in parallel, we only create one
# tag for that folder. this happens when the CondstoreFolderSyncEngine
# checks for UID changes.
# patching the heartbeat clear means that we force the first greenlet to
# wait around (there is a deleted folder in folder_name_mapping), thereby
# assuring that the second greenlet will overtake it and force any
# potential race condition around tag creation.
def clear_heartbeat_patch(w, x, y, z):
gevent.sleep(1)
monkeypatch.setattr('inbox.heartbeat.store.HeartbeatStore.remove_folders',
clear_heartbeat_patch)
log = get_logger()
group = Group()
with mailsync_session_scope() as db_session:
group.spawn(save_folder_names, log, ACCOUNT_ID,
folder_name_mapping, db_session)
with mailsync_session_scope() as db_session:
group.spawn(save_folder_names, log, ACCOUNT_ID,
folder_name_mapping, db_session)
group.join()
with mailsync_session_scope() as db_session:
account = db_session.query(Account).get(ACCOUNT_ID)
random_tags = db_session.query(Tag).filter_by(
namespace_id=account.namespace.id,
name='random')
assert random_tags.count() == 1
class PbClient:
def __init__(self, ip, port):
self.addr = (ip, int(port))
self.callback_dict = {}
self.greenlets = Group()
self.soc = socket()
self.interval = 1/200
self.received_data = b''
def connect(self):
self.soc.connect(self.addr)
self.greenlets.spawn(self.on_message)
def send(self, data):
self.soc.send(data)
def close(self):
try:
self.greenlets.kill()
self.soc.close()
except:
pass
finally:
self.soc = None
def on_message(self):
try:
while self.soc:
temp = self.soc.recv(2048)
# gevent.sleep(self.interval)
if temp:
self.received_data += temp
self.handle_message()
except EnvironmentError:
pass
except:
raise
def handle_message(self):
try:
if len(self.received_data) < HEAD_SIZE:
return
data = self.get_pack_stream()
data_dict = DotDict(DataStruct.parse(data))
cmd_name = CommandMapping(data_dict['CommandID'])
self.callback_dict[cmd_name] = data_dict
logger.info("Receive data from {}:{}".format(cmd_name, data_dict))
self.handle_message()
except ValueError:
logger.warn("Receive unknown commandID: {}".format(cmd_name))
except TypeError as e:
logger.error("Receive invalid data:", exc_info=True)
def get_pack_stream(self):
'''获取单个完整包的数据流'''
size_stream = self.received_data[1:4]
pack_size = pack_size_struct.parse(size_stream)
pack_stream = self.received_data[:pack_size]
self.received_data = self.received_data[pack_size:]
return pack_stream
def _search(responder, id, search, max_results):
todo = dict()
group = Group()
for ctx in search.contexts:
if len(ctx.results) > max_results/len(search.contexts):
continue
todo[ctx.name] = 0
group.spawn(_run, responder, search, ctx, todo)
group.join()
for ctx in search.contexts[:]:
if not ctx.results:
search.remove_ctx(ctx)
results = list()
while search.contexts and (max_results is None or len(results) < max_results):
ctx = search.next_result_ctx()
if not ctx.results:
break
results.append(ctx.results.pop(0))
display = search.display
for ctx in search.contexts[:]:
if ctx.next is None:
search.remove_ctx(ctx)
if search.more:
cache[search.id] = search
elif search.id in cache:
del cache[search.id]
return dict(id=id, search_id=search.id, display=display, more=search.more, results=results)
def patch_all(timeout=180, external_loaded=True, source_complete_callback=None):
with patch_all_lock:
# check config urls
log.debug('checking config urls')
todo = list()
for source in sources.values():
config_url = source.get_config_url()
if config_url is not None and config_url not in todo:
todo.append(config_url)
group = Group()
for config_url in todo:
g = group.spawn(config_url.update)
patch_group.add(g)
group.join()
log.debug('updating repos')
# check for updates
patches = list()
for source in sources.values():
if source.enabled:
def _patch(patches, source, timeout):
try:
patch_one(patches, source, timeout)
finally:
if source_complete_callback is not None:
source_complete_callback(source)
g = group.spawn(_patch, patches, source, timeout)
patch_group.add(g)
group.join()
finalize_patches(patches, external_loaded=external_loaded)
class MasterRunner(Runner):
def __init__(self, host, port):
super().__init__(host, port)
self.server = rpc.Server(self.host, self.port)
self.slave = {}
self.greenlet = Group()
self.greenlet.spawn(self.slave_listener)
# self.slave_listener()
def slave_listener(self):
while True:
msg = self.server.recv()
if msg.type == 'slave_ready':
id = msg.node_id
self.slave[id] = id
logger.info(
"Client %r reported as ready. Currently %i slaves ready to emit."
% (id, len(self.slave)))
self.server.send(Message('slave_stop', None, None))
elif msg.type == 'slave_stats':
logger.info(msg.data)
elif msg.type == 'slave_complete':
logger.info("=====================================")
self.server.send(Message('slave_stop', None, None))
elif msg.type == 'slave_start':
pass
elif msg.type == 'slave_stop':
del self.slave[msg.node_id]
logger.info("Removing %s from running slaves" % (msg.node_id))
if len(self.slave) == 0:
self.state = 'stopped'
class SlaveLocustRunner(DistributedLocustRunner):
def __init__(self, *args, **kwargs):
super(SlaveLocustRunner, self).__init__(*args, **kwargs)
self.client_id = socket.gethostname() + "_" + md5(str(time() + random.randint(0,10000))).hexdigest()
self.client = rpc.Client(self.master_host)
self.greenlet = Group()
self.greenlet.spawn(self.worker).link_exception()
self.client.send(Message("client_ready", None, self.client_id))
self.greenlet.spawn(self.stats_reporter).link_exception()
# register listener for when all locust users have hatched, and report it to the master node
def on_hatch_complete(count):
self.client.send(Message("hatch_complete", {"count":count}, self.client_id))
events.hatch_complete += on_hatch_complete
# register listener that adds the current number of spawned locusts to the report that is sent to the master node
def on_report_to_master(client_id, data):
data["user_count"] = self.user_count
events.report_to_master += on_report_to_master
# register listener that sends quit message to master
def on_quitting():
self.client.send(Message("quit", None, self.client_id))
events.quitting += on_quitting
# register listener thats sends locust exceptions to master
def on_locust_error(locust, e, tb):
formatted_tb = "".join(traceback.format_tb(tb))
self.client.send(Message("exception", {"msg" : str(e), "traceback" : formatted_tb}, self.client_id))
events.locust_error += on_locust_error
def worker(self):
while True:
msg = self.client.recv()
if msg.type == "hatch":
self.client.send(Message("hatching", None, self.client_id))
job = msg.data
self.hatch_rate = job["hatch_rate"]
#self.num_clients = job["num_clients"]
self.num_requests = job["num_requests"]
self.host = job["host"]
self.hatching_greenlet = gevent.spawn(lambda: self.start_hatching(locust_count=job["num_clients"], hatch_rate=job["hatch_rate"]))
elif msg.type == "stop":
self.stop()
self.client.send(Message("client_stopped", None, self.client_id))
self.client.send(Message("client_ready", None, self.client_id))
def stats_reporter(self):
while True:
data = {}
events.report_to_master.fire(self.client_id, data)
try:
self.client.send(Message("stats", data, self.client_id))
except:
logger.error("Connection lost to master server. Aborting...")
break
gevent.sleep(SLAVE_REPORT_INTERVAL)
class SlaveLocustRunner(DistributedLocustRunner):
def __init__(self, *args, **kwargs):
super(SlaveLocustRunner, self).__init__(*args, **kwargs)
self.client_id = socket.gethostname() + "_" + md5(str(time() + random.randint(0,10000))).hexdigest()
self.client = zmqrpc.Client(self.master_host)
self.greenlet = Group()
self.greenlet.spawn(self.worker).link_exception()
self.client.send({"type":"client_ready", "data":self.client_id})
self.greenlet.spawn(self.stats_reporter).link_exception()
# register listener for when all locust users have hatched, and report it to the master node
def on_hatch_complete(count):
self.client.send({"type":"hatch_complete", "data":{"client_id":self.client_id, "count":count}})
events.hatch_complete += on_hatch_complete
# register listener that adds the current number of spawned locusts to the report that is sent to the master node
def on_report_to_master(client_id, data):
data["user_count"] = self.user_count
events.report_to_master += on_report_to_master
# register listener that sends quit message to master
def on_quitting():
self.client.send({"type":"quit", "data":self.client_id})
events.quitting += on_quitting
def worker(self):
while True:
msg = self.client.recv()
if msg["type"] == "hatch":
self.client.send({"type":"hatching", "data":self.client_id})
job = msg["data"]
self.hatch_rate = job["hatch_rate"]
#self.num_clients = job["num_clients"]
self.num_requests = job["num_requests"]
self.host = job["host"]
self.hatching_greenlet = gevent.spawn(lambda: self.start_hatching(locust_count=job["num_clients"], hatch_rate=job["hatch_rate"]))
elif msg["type"] == "stop":
self.stop()
self.client.send({"type":"client_stopped", "data":self.client_id})
self.client.send({"type":"client_ready", "data":self.client_id})
def stats_reporter(self):
while True:
data = {}
events.report_to_master.fire(self.client_id, data)
report = {
"client_id": self.client_id,
"data": data,
}
try:
self.client.send({"type":"stats", "data":report})
except:
logger.error("Connection lost to master server. Aborting...")
break
gevent.sleep(SLAVE_REPORT_INTERVAL)
def spawn_greenlets(conf):
"""Some sugar to wrap up all of your greenlets."""
group = Group()
for args in conf:
group.spawn(*args)
try:
while True:
gevent.sleep(1)
except KeyboardInterrupt:
pass
def execute(l):
from gevent.pool import Group
letters = string.ascii_lowercase
key = random.randrange(0, 1000)
second_key = random.randrange(0, 1000)
value = ''.join(random.choice(letters) for i in range(10))
group = Group()
group.spawn(lambda: l.client.get(f"/set/{key}/{value}"))
group.spawn(lambda: l.client.get(f"/get/{second_key}"))
group.join()
def terminate():
g = Group()
for torrent in torrents.values():
g.spawn(torrent.save_resume_data)
try:
g.join(timeout=5)
except:
pass
for torrent in torrents.values():
torrent.remove()
def terminate():
g = Group()
for torrent in torrents.values():
g.spawn(torrent.save_resume_data)
try:
g.join(timeout=5)
except:
pass
for torrent in torrents.values():
torrent.remove()
def loadConfigSchema(self):
def config():
self.client.get(f"{self.dataset}{API}/config", stream=True).close()
def schema():
self.client.get(f"{self.dataset}{API}/schema", stream=True).close()
group = Group()
group.spawn(config)
group.spawn(schema)
group.join()
def loadObsAnnotations(self):
def obs_annotation(name):
self.client.get(
f"{self.dataset}{API}/annotations/obs?annotation-name={name}",
headers={"Accept": "application/octet-stream"},
stream=True,
).close()
obs_names = self.parent.obs_annotation_names()
group = Group()
for name in obs_names:
group.spawn(obs_annotation, name)
group.join()
def test_kill_processlet_group(proc):
group = Group()
group.greenlet_class = lets.Processlet
group.spawn(raise_when_killed)
group.spawn(raise_when_killed)
group.spawn(raise_when_killed)
group.join(0)
assert len(proc.children()) == 3
group.kill()
assert len(proc.children()) == 0
for job in group:
with pytest.raises(Killed):
job.get()
assert job.exit_code == 1
def _main():
user_crawler_group = Group()
for _ in xrange(GREENLET_COUNT):
user_crawler_group.spawn(analyze)
with open('ids.txt') as FILE:
for line in FILE:
id = line.strip()
cursor.execute('SELECT COUNT(1) as total_count FROM tb_xweibo_user_info WHERE uid = %s' % id)
result = cursor.fetchone()
if not result['total_count']:
users_fetch_queue.put(id)
user_crawler_group.join()
def get_multihoster_account(task, multi_match, file):
if not account.config.use_useraccounts:
print "multihoster off, use_useraccounts false"
return
group = Group()
for pool in account.manager.values():
for acc in pool:
if acc.multi_account:
from . import manager
acc.hoster = manager.find_by_name(acc.name)
group.spawn(acc.boot)
group.join()
accounts = []
best_weight = 0
hostname = file.split_url.host
for pool in account.manager.values():
for acc in pool:
if acc._private_account:
continue
if not acc.multi_account:
continue
if hasattr(acc, 'premium') and not acc.premium:
continue
print acc.premium
if not multi_match(acc, hostname):
continue
try:
weight = acc.weight
except gevent.GreenletExit:
print "greenlet exit"
continue
if weight > best_weight:
accounts = []
best_weight = weight
bisect.insort(accounts, (acc.get_task_pool(task).full() and 1
or 0, len(accounts), acc))
if accounts:
return accounts[0][2]
"""try:
file.log.info('trying multihoster {}'.format(acc.name))
acc.hoster.get_download_context(file)
except gevent.GreenletExit:
raise
except BaseException as e:
log.exception(e)"""
else:
print "multi: no accounts found"
class Manager(object):
def __init__(self, config_addr, keyfile,
certfile, cacerts, backlog=10):
if isinstance(config_addr, basestring):
ip, port = config_addr.split(':')
config_addr = (ip, int(port))
self.keyfile = keyfile
self.certfile = certfile
self.cacerts = cacerts
self.config_addr = config_addr
self.backlog = backlog
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.end_evt = Event()
self.clients = Group()
self.redirectors = {}
self.msg_processors = {
'redirect': self.m_redirect,
'list_redirect': self.m_list_redirect,
'drop_redirect': self.m_drop_redirect,
'shutdown': self.m_shutdown,
}
logging.info('manager initialized')
def run(self):
logging.info('manager start to run')
self.sock.bind(self.config_addr)
logging.info('manager bind to: %s:%d' % self.config_addr)
self.sock.listen(self.backlog)
accept_let = gevent.spawn(self.accept_let)
self.end_evt.wait()
logging.info('shutdown evt recved')
accept_let.kill()
self.clients.kill()
def accept_let(self):
while True:
sock, addr = self.sock.accept()
try:
sock = ssl.wrap_socket(sock, keyfile=self.keyfile,
certfile=self.certfile, server_side=True,
cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.cacerts)
except ssl.SSLError, e:
print e
continue
self.clients.spawn(self.client_let, sock, addr)
def test_gevent(num, depth, period, settle):
global SWITCH_COUNT
import gevent
from gevent.pool import Group
from gevent.hub import sleep
horde = Group()
for x in xrange(num):
horde.spawn(stack_filler, depth, sleep)
gevent.sleep(settle)
print("settle period over, {:.2f} sw/sec, testing".format(SWITCH_COUNT/(1.0*settle)))
SWITCH_COUNT=0
gevent.sleep(period)
print("testing period over, {:.2f} sw/sec".format(SWITCH_COUNT/(1.0*period)))
horde.kill()
def get_multihoster_account(task, multi_match, file):
if not account.config.use_useraccounts:
print "multihoster off, use_useraccounts false"
return
group = Group()
for pool in account.manager.values():
for acc in pool:
if acc.multi_account:
from . import manager
acc.hoster = manager.find_by_name(acc.name)
group.spawn(acc.boot)
group.join()
accounts = []
best_weight = 0
hostname = file.split_url.host
for pool in account.manager.values():
for acc in pool:
if acc._private_account:
continue
if not acc.multi_account:
continue
if hasattr(acc, 'premium') and not acc.premium:
continue
print acc.premium
if not multi_match(acc, hostname):
continue
try:
weight = acc.weight
except gevent.GreenletExit:
print "greenlet exit"
continue
if weight > best_weight:
accounts = []
best_weight = weight
bisect.insort(accounts, (acc.get_task_pool(task).full() and 1 or 0, len(accounts), acc))
if accounts:
return accounts[0][2]
"""try:
file.log.info('trying multihoster {}'.format(acc.name))
acc.hoster.get_download_context(file)
except gevent.GreenletExit:
raise
except BaseException as e:
log.exception(e)"""
else:
print "multi: no accounts found"
def _enqueue_children(resources, seed, parent):
threads = Group()
for resource in [r for r in resources if r.parent == parent.name]:
thread = threads.spawn(_process_resource, resources, seed, resource)
thread.link_exception(_create_error_handler(resource.collection))
threads.join()
def _push_to_target(self, targets):
"""Get a batch of elements from the queue, and push it to the targets.
This function returns True if it proceeded all the elements in
the queue, and there isn't anything more to read.
"""
if self.queue.empty():
return 0 # nothing
batch = []
pushed = 0
# collecting a batch
while len(batch) < self.batch_size:
item = self.queue.get()
if item == 'END':
pushed += 1 # the 'END' item
break
batch.append(item)
if len(batch) != 0:
greenlets = Group()
for plugin in targets:
green = greenlets.spawn(self._put_data, plugin, batch)
green.link_exception(partial(self._error,
exception.InjectError, plugin))
greenlets.join()
pushed += len(batch)
return pushed
def handle_tcp(self, sock, remote):
fdset = [sock, remote]
tasks = Group()
prev = [0, 0]
sock_switch = remote_switch = 0
sock_counter = remote_counter = 0
sock_count = remote_count = 0
sock_size = array.array('i', [0])
remote_size = array.array('i', [0])
while True:
r, w, e = select.select(fdset, [], [])
# Problem is knowing beforehand when the socket is going to switch to joinall remainding tasks
# FIONREAD will check size of available bytes of the socket to catch the last send()/recv()
if sock in r:
if sock_switch == 0:
fcntl.ioctl(sock, termios.FIONREAD, sock_size, True)
sock_count = ceil(sock_size[0] / float(BUF_SIZE))
print("sock", sock_size[0], sock_count)
sock_switch = 1
delay = self.calculate_delay(prev)
sock_buf = sock.recv(BUF_SIZE)
#print(sock_buf)
if sock_buf is None or sock_buf == "": break
tasks.spawn(self.delay_message, sock_buf, delay, remote)
sock_counter += 1
if remote in r:
if remote_switch == 0:
fcntl.ioctl(remote, termios.FIONREAD, remote_size, True)
remote_count = ceil(remote_size[0] / float(BUF_SIZE))
print("remote", remote_size[0], remote_count)
remote_switch = 1
delay = self.calculate_delay(prev)
remote_buf = remote.recv(BUF_SIZE)
#print(remote_buf)
if remote_buf is None or remote_buf == "": break
tasks.spawn(self.delay_message, remote_buf, delay, sock)
remote_counter += 1
# Wait for last task before switching socket
if sock_count == sock_counter and sock_switch == 1:
print("joiningsocks")
tasks.join()
sock_counter = sock_switch = 0
if remote_count == remote_counter and remote_switch == 1:
print("joiningremote")
tasks.join()
remote_counter = remote_switch = 0
def loadBootstrapData(self):
def layout():
self.client.get(
f"{self.dataset}{API}/layout/obs", headers={"Accept": "application/octet-stream"}, stream=True
).close()
def varAnnotationIndex():
self.client.get(
f"{self.dataset}{API}/annotations/var?annotation-name={self.parent.var_index_name()}",
headers={"Accept": "application/octet-stream"},
stream=True,
).close()
group = Group()
group.spawn(layout)
group.spawn(varAnnotationIndex)
group.join()
class GeventExecutor(wrapper.ExecutorBase):
def __init__(self):
super().__init__()
self.group = Group()
self.channel: ThreadsafeChannel[Callable[[],
None]] = ThreadsafeChannel()
self.spawn_work_greenlet = gevent.spawn(self._spawn_work)
def __del__(self):
super().__del__()
self.spawn_work_greenlet.kill()
def execute(self, func: Callable[[], None]):
self.channel.put(func)
def _spawn_work(self):
while True:
self.group.spawn(self.channel.get())
class TCPClient():
def __init__(self):
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._group = Group()
self._send_buffer = Queue()
self._recv_buffer = Queue()
def connect(self, address):
self._socket.connect(address)
self._group.spawn(self._recv_loop)
self._group.spawn(self._send_loop)
def disconnect(self):
self._group.kill()
self._socket.close()
self._group.join()
def _recv_loop(self):
buf = ""
u4 = lambda x: unpack("<I", x)[0]
while True:
data = self._socket.recv(8192)
buf += data
while len(buf) > 0:
length = u4(buf[:4])
if len(buf) - 4 < length:
break
self._recv_buffer.put(buf[4:4 + length])
buf = buf[4 + length:]
def _send_loop(self):
while True:
data = self._send_buffer.get()
self._socket.sendall(data)
def write(self, data):
self._send_buffer.put(data)
def get_packet(self):
return self._recv_buffer.get()
def execute(resources, seed=None):
threads = Group()
for resource in [r for r in resources if not r.parent]:
thread = threads.spawn(_process_resource, resources, seed, resource)
thread.link_exception(_create_error_handler(resource.collection))
threads.join()
if len(_errors):
sys.exit(1)
class Dispatcher(gevent.Greenlet):
def __init__(self, collector, publisher, quiet=False):
super(Dispatcher, self).__init__()
self.collector = collector
self.publisher = publisher
self.quiet = quiet
self.greenlets = Group()
self.channel = gevent.queue.Queue(0)
self._keep_going = True
def _run(self):
self.greenlets.spawn(self.__collect)
self.greenlets.spawn(self.__publish)
self.greenlets.join()
def kill(self, exception=gevent.GreenletExit, **kwargs):
self._keep_going = False
self.greenlets.kill()
super(Dispatcher, self).kill(exception=exception, **kwargs)
def __collect(self):
while self._keep_going:
message = self.collector.recv_multipart()
self.channel.put(message)
gevent.sleep()
def __publish(self):
while self._keep_going:
message = self.channel.get()
if not self.quiet:
# message is assumed to be a tuple of: (topic, record_json)
topic,record_json = message
topic = topic.decode()
name_and_level = topic[len(zerolog.stream_prefix):]
logger_name,level_name = name_and_level.split(':')
logger = zerolog.getLocalLogger(logger_name)
if logger.isEnabledFor(logging.getLevelName(level_name)):
# inject log record into local logger
record_dict = json.loads(record_json.decode())
record = logging.makeLogRecord(record_dict)
logger.handle(record)
self.publisher.send_multipart(message)
gevent.sleep()
def _search(responder, id, search, max_results):
responder._sent = False
todo = dict()
group = Group()
groups[id] = [group]
for ctx in search.contexts:
groups[id].append(ctx.thumb_pool)
if len(ctx.results) > max_results/len(search.contexts):
continue
todo[ctx.name] = 0
group.spawn(_run, responder, search, ctx, todo)
group.join()
for ctx in search.contexts[:]:
if not ctx.results:
search.remove_ctx(ctx)
results = list()
while search.contexts and (max_results is None or len(results) < max_results):
ctx = search.next_result_ctx()
if not ctx.results:
break
results.append(ctx.results.pop(0))
display = search.display
for ctx in search.contexts[:]:
if ctx.next is None:
search.remove_ctx(ctx)
if search.more:
cache[search.id] = search
elif search.id in cache:
del cache[search.id]
payload = dict(id=id, search_id=search.id, display=display, more=search.more, results=results)
responder.send(payload=payload)
responder._sent = True
try:
del groups[id]
except KeyError:
pass
def _update(self):
found_sources = list()
resp = requests.get(self.url, stream=True)
try:
resp.raise_for_status()
data = yaml.load(resp.raw)
finally:
resp.close()
assert len(data.keys()) > 0
group = Group()
def _add_source(url):
try:
source = add_source(url, self.url)
except:
self.log.warning('error adding new repo {}'.format(url))
else:
found_sources.append(source)
for name, url in data.iteritems():
try:
Url(url)
except:
self.log.warning('invalid patch source entry: {}'.format(url))
try:
source = sources[name]
except KeyError:
self.log.info('adding new repo {}'.format(url))
group.spawn(_add_source, url)
else:
found_sources.append(source)
if source.url != url:
source.log.info('changing url to {}'.format(url))
with transaction:
source.url = url
source.unlink()
group.join()
for source in sources.values():
if source.config_url == self.url and source not in found_sources:
source.log.info('erasing repo')
source.delete(True)
def run(fqn, concurrency=1, numruns=1):
""" Runs a test.
* fnq: fully qualified name
* concurrency: number of concurrent runs
* numruns: number of run per concurrent
"""
set_global_stream('stdout', total=concurrency * numruns)
test = resolve_name(fqn)
klass = test.im_class
ob = klass(test.__name__)
test_result = unittest.TestResult()
group = Group()
for i in range(concurrency):
group.spawn(_run, i, ob, test_result, numruns)
group.join()
return test_result
def test_ungraceful_close_in_greenlet(self):
group = Group()
UnaryCallWithSleep = self._channel.unary_unary(
_UNARY_CALL_METHOD_WITH_SLEEP,
request_serializer=messages_pb2.SimpleRequest.SerializeToString,
response_deserializer=messages_pb2.SimpleResponse.FromString,
)
greenlet = group.spawn(self._run_client, UnaryCallWithSleep)
# release loop so that greenlet can take control
gevent.sleep()
group.killone(greenlet)
self.assertFalse(self._unhandled_exception, "Unhandled GreenletExit")
def get_best(self, task, file):
with self.lock:
if config.use_useraccounts:
all_accounts = self
else:
all_accounts = [a for a in self if a._private_account]
group = Group()
for account in all_accounts:
group.spawn(account.boot)
group.join()
all_accounts = [a for a in all_accounts if a._private_account or (a.enabled and a.last_error is None)]
accounts = []
best_weight = 0
for account in all_accounts:
if file is not None and not account.match(file):
continue
try:
weight = account.weight
except gevent.GreenletExit:
continue
if weight is None or weight < best_weight:
continue
if weight > best_weight:
accounts = []
best_weight = weight
bisect.insort(accounts, (account.get_task_pool(task).full() and 1 or 0, len(accounts), account))
if accounts:
return accounts[0][2]
if len(all_accounts) > 0:
#self.log.warning('found no account. returning first one...')
return all_accounts[0]
else:
self.log.info('found no account. creating a "free" account')
account = self.add(_private_account=True)
account.boot()
return account
def test_kill_greenlet_with_generic_exception(self):
group = Group()
UnaryCallWithSleep = self._channel.unary_unary(
_UNARY_CALL_METHOD_WITH_SLEEP,
request_serializer=messages_pb2.SimpleRequest.SerializeToString,
response_deserializer=messages_pb2.SimpleResponse.FromString,
)
greenlet = group.spawn(self._run_client, UnaryCallWithSleep)
# release loop so that greenlet can take control
gevent.sleep()
group.killone(greenlet, exception=Exception)
self.assertFalse(self._unhandled_exception, "Unhandled exception")
self.assertRaises(Exception, greenlet.get)
def test_graceful_close_in_greenlet(self):
group = Group()
stub = test_pb2_grpc.TestServiceStub(self._channel)
greenlet = group.spawn(self._run_client, stub.UnaryCall)
# release loop so that greenlet can take control
gevent.sleep()
self._channel.close()
group.killone(greenlet)
self.assertFalse(self._unhandled_exception, "Unhandled GreenletExit")
try:
greenlet.get()
except Exception as e: # pylint: disable=broad-except
self.fail(f"Unexpected exception in greenlet: {e}")
class GeventExecutor(Executor):
def __init__(self):
self.group = Group()
self.channel: GeventChannel[
Tuple[Callable, List[Any], Dict[str, Any]]
] = GeventChannel()
self.spawn_work_greenlet = gevent.spawn(self._spawn_work)
def __del__(self):
self.spawn_work_greenlet.kill()
def wrap(self, fn: Func) -> Func:
@functools.wraps(fn)
def wrapper(*args, **kwargs):
self.channel.put((fn, args, kwargs))
return cast(Func, wrapper)
def _spawn_work(self):
while True:
(fn, args, kwargs) = self.channel.get()
self.group.spawn(fn, *args, **kwargs)
class SyncStack:
class SyncLayer:
def __init__(self, path):
self.path = path
self.delta = None
def __init__(self):
self.layers = []
self.latest_tag = 0
self.sync_group = Group()
def append(self, path):
self.layers.append(self.SyncLayer(path))
def has_unknown_delta(self):
return self.latest_tag < len(self.layers)
def need_sync(self, worker):
return worker.sync_tag < self.latest_tag
def update_delta(self, worker):
assert self.has_unknown_delta()
assert worker.sync_tag == self.latest_tag
layer = self.layers[self.latest_tag]
layer.delta = worker.calc_dir_delta(layer.path)
self.latest_tag += 1
def start_sync(self, worker):
def _sync():
while worker.sync_tag < self.latest_tag:
layer = self.layers[worker.sync_tag]
worker.sync_with_delta(layer.delta, layer.path)
worker.set_syncing(False)
worker.set_syncing(True)
self.sync_group.spawn(_sync)
def stop(self):
self.sync_group.kill()
def api_calls(self):
# Run all the API calls in parallel
group = Group()
if self.isNotFerry:
group.spawn(self.realtime_polling)
group.spawn(self.map_api)
group.spawn(self.line_api)
group.join()
def _search(responder, id, search, max_results):
todo = dict()
group = Group()
for ctx in search.contexts:
if len(ctx.results) > max_results / len(search.contexts):
continue
todo[ctx.name] = 0
group.spawn(_run, responder, search, ctx, todo)
group.join()
for ctx in search.contexts[:]:
if not ctx.results:
search.remove_ctx(ctx)
results = list()
while search.contexts and (max_results is None
or len(results) < max_results):
ctx = search.next_result_ctx()
if not ctx.results:
break
results.append(ctx.results.pop(0))
display = search.display
for ctx in search.contexts[:]:
if ctx.next is None:
search.remove_ctx(ctx)
if search.more:
cache[search.id] = search
elif search.id in cache:
del cache[search.id]
return dict(id=id,
search_id=search.id,
display=display,
more=search.more,
results=results)
def run_troller():
youtube_interest_q = Queue()
detector_input_q = Queue()
bad_video_q = Queue()
tumblr_q = Queue()
reddit_q = Queue()
debug_q = Queue()
conf = [
(scrape_reddit, youtube_interest_q),
(download_youtube_thumbs, youtube_interest_q, detector_input_q),
(detect_bad_videos, detector_input_q, bad_video_q),
(debug_results, debug_q),
(post_to_tumblr, tumblr_q),
(post_reddit_response, reddit_q),
(bad_video_fanout, bad_video_q, (tumblr_q, reddit_q, debug_q)),
]
group = Group()
for args in conf:
group.spawn(*args)
try:
while True:
gevent.sleep(1)
except KeyboardInterrupt:
pass
def ready(role):
queue = Group()
while True:
info = ScaleInfo(role)
_scale_ctx_stack.push(info)
if config.check_func is not None:
queue.spawn(action, role, config.check_func())
else:
queue.spawn(action, role, check_cpu_utilization(role))
queue.spawn(gevent.sleep, role.cooltime*60)
queue.join()
_scale_ctx_stack.pop()
class Processor(object):
def __init__(self):
self.group = Group()
self.greenlets = []
def spawn(self, func, *args, **kwargs):
g = self.group.spawn(func, *args, **kwargs)
self.greenlets.append(g)
def join(self):
self.group.join()
def values(self):
gs, self.greenlets = self.greenlets, []
return [g.value for g in gs]
def empty(self):
return not bool(self.greenlets)
def _run_phase(self, phase, start_date, end_date):
phase, sources, targets = phase
logger.info('Running phase %r' % phase)
self._reset_counters()
self._start_transactions(targets)
self.database.start_transaction()
try:
greenlets = Group()
# each callable will push its result in the queue
for source in sources:
exists = self.database.exists(source, start_date, end_date)
if exists and not self.force:
logger.info('Already done: %s, %s to %s' % (
source.get_id(), start_date, end_date))
continue
green = greenlets.spawn(self._get_data, source,
start_date, end_date)
green.link_value(partial(self._log_transaction, source,
start_date, end_date))
green.link_exception(partial(self._error,
exception.ExtractError, source))
# looking at the queue
pushed = 0
while len(greenlets) > 0 or self.queue.qsize() > 0:
gevent.sleep(0)
pushed += self._push_to_target(targets)
# let's see if we have some errors
if len(self.errors) > 0:
# yeah! we need to rollback
# XXX later we'll do a source-by-source rollback
raise exception.RunError(self.errors)
except Exception:
self._rollback_transactions(targets)
self.database.rollback_transaction()
raise
else:
self._commit_transactions(targets)
self.database.commit_transaction()
class LocustRunner(object):
def __init__(self, locust_classes, hatch_rate, num_clients, num_requests=None, host=None):
self.locust_classes = locust_classes
self.hatch_rate = hatch_rate
self.num_clients = num_clients
self.num_requests = num_requests
self.host = host
self.locusts = Group()
self.state = STATE_INIT
self.hatching_greenlet = None
self.exceptions = {}
self.stats = global_stats
# register listener that resets stats when hatching is complete
def on_hatch_complete(count):
self.state = STATE_RUNNING
logger.info("Resetting stats\n")
self.stats.reset_all()
events.hatch_complete += on_hatch_complete
@property
def request_stats(self):
return self.stats.entries
@property
def errors(self):
return self.stats.errors
@property
def user_count(self):
return len(self.locusts)
def weight_locusts(self, amount, stop_timeout = None):
"""
Distributes the amount of locusts for each WebLocust-class according to it's weight
returns a list "bucket" with the weighted locusts
"""
bucket = []
weight_sum = sum((locust.weight for locust in self.locust_classes if locust.task_set))
for locust in self.locust_classes:
if not locust.task_set:
warnings.warn("Notice: Found Locust class (%s) got no task_set. Skipping..." % locust.__name__)
continue
if self.host is not None:
locust.host = self.host
if stop_timeout is not None:
locust.stop_timeout = stop_timeout
# create locusts depending on weight
percent = locust.weight / float(weight_sum)
num_locusts = int(round(amount * percent))
bucket.extend([locust for x in xrange(0, num_locusts)])
return bucket
def spawn_locusts(self, spawn_count=None, stop_timeout=None, wait=False):
if spawn_count is None:
spawn_count = self.num_clients
if self.num_requests is not None:
self.stats.max_requests = self.num_requests
bucket = self.weight_locusts(spawn_count, stop_timeout)
spawn_count = len(bucket)
if self.state == STATE_INIT or self.state == STATE_STOPPED:
self.state = STATE_HATCHING
self.num_clients = spawn_count
else:
self.num_clients += spawn_count
logger.info("Hatching and swarming %i clients at the rate %g clients/s..." % (spawn_count, self.hatch_rate))
occurence_count = dict([(l.__name__, 0) for l in self.locust_classes])
def hatch():
sleep_time = 1.0 / self.hatch_rate
while True:
if not bucket:
logger.info("All locusts hatched: %s" % ", ".join(["%s: %d" % (name, count) for name, count in occurence_count.iteritems()]))
events.hatch_complete.fire(self.num_clients)
return
locust = bucket.pop(random.randint(0, len(bucket)-1))
occurence_count[locust.__name__] += 1
def start_locust(_):
try:
locust().run()
except GreenletExit:
pass
new_locust = self.locusts.spawn(start_locust, locust)
if len(self.locusts) % 10 == 0:
logger.debug("%i locusts hatched" % len(self.locusts))
gevent.sleep(sleep_time)
hatch()
if wait:
self.locusts.join()
logger.info("All locusts dead\n")
def kill_locusts(self, kill_count):
"""
Kill a kill_count of weighted locusts from the Group() object in self.locusts
"""
bucket = self.weight_locusts(kill_count)
kill_count = len(bucket)
self.num_clients -= kill_count
logger.info("Killing %i locusts" % kill_count)
dying = []
for g in self.locusts:
for l in bucket:
if l == g.args[0]:
dying.append(g)
bucket.remove(l)
break
for g in dying:
self.locusts.killone(g)
events.hatch_complete.fire(self.num_clients)
def start_hatching(self, locust_count=None, hatch_rate=None, wait=False):
if self.state != STATE_RUNNING and self.state != STATE_HATCHING:
self.stats.clear_all()
self.stats.start_time = time()
self.exceptions = {}
# Dynamically changing the locust count
if self.state != STATE_INIT and self.state != STATE_STOPPED:
self.state = STATE_HATCHING
if self.num_clients > locust_count:
# Kill some locusts
kill_count = self.num_clients - locust_count
self.kill_locusts(kill_count)
elif self.num_clients < locust_count:
# Spawn some locusts
if hatch_rate:
self.hatch_rate = hatch_rate
spawn_count = locust_count - self.num_clients
self.spawn_locusts(spawn_count=spawn_count)
else:
if hatch_rate:
self.hatch_rate = hatch_rate
if locust_count:
self.spawn_locusts(locust_count, wait=wait)
else:
self.spawn_locusts(wait=wait)
def stop(self):
# if we are currently hatching locusts we need to kill the hatching greenlet first
if self.hatching_greenlet and not self.hatching_greenlet.ready():
self.hatching_greenlet.kill(block=True)
self.locusts.kill(block=True)
self.state = STATE_STOPPED
def log_exception(self, node_id, msg, formatted_tb):
key = hash(formatted_tb)
row = self.exceptions.setdefault(key, {"count": 0, "msg": msg, "traceback": formatted_tb, "nodes": set()})
row["count"] += 1
row["nodes"].add(node_id)
self.exceptions[key] = row
class MasterLocustRunner(DistributedLocustRunner):
def __init__(self, *args, **kwargs):
super(MasterLocustRunner, self).__init__(*args, **kwargs)
class SlaveNodesDict(dict):
def get_by_state(self, state):
return [c for c in self.itervalues() if c.state == state]
@property
def ready(self):
return self.get_by_state(STATE_INIT)
@property
def hatching(self):
return self.get_by_state(STATE_HATCHING)
@property
def running(self):
return self.get_by_state(STATE_RUNNING)
self.clients = SlaveNodesDict()
self.client_stats = {}
self.client_errors = {}
self._request_stats = {}
self.server = rpc.Server()
self.greenlet = Group()
self.greenlet.spawn(self.client_listener).link_exception(receiver=self.noop)
# listener that gathers info on how many locust users the slaves has spawned
def on_slave_report(client_id, data):
self.clients[client_id].user_count = data["user_count"]
events.slave_report += on_slave_report
# register listener that sends quit message to slave nodes
def on_quitting():
self.quit()
events.quitting += on_quitting
def noop(self, *args, **kw):
pass
@property
def user_count(self):
return sum([c.user_count for c in self.clients.itervalues()])
def start_hatching(self, locust_count, hatch_rate):
self.num_clients = locust_count
slave_num_clients = locust_count / ((len(self.clients.ready) + len(self.clients.running)) or 1)
slave_hatch_rate = float(hatch_rate) / ((len(self.clients.ready) + len(self.clients.running)) or 1)
logger.info("Sending hatch jobs to %i ready clients" % (len(self.clients.ready) + len(self.clients.running)))
if not (len(self.clients.ready)+len(self.clients.running)):
logger.warning("You are running in distributed mode but have no slave servers connected. Please connect slaves prior to swarming.")
return
if self.state != STATE_RUNNING and self.state != STATE_HATCHING:
self.stats.clear_all()
self.exceptions = {}
for client in self.clients.itervalues():
data = {"hatch_rate":slave_hatch_rate, "num_clients":slave_num_clients, "num_requests": self.num_requests, "host":self.host, "stop_timeout":None}
self.server.send(Message("hatch", data, None))
self.stats.start_time = time()
self.state = STATE_HATCHING
def stop(self):
for client in self.clients.hatching + self.clients.running:
self.server.send(Message("stop", None, None))
def quit(self):
for client in self.clients.itervalues():
self.server.send(Message("quit", None, None))
self.greenlet.kill(block=True)
def client_listener(self):
while True:
msg = self.server.recv()
if msg.type == "client_ready":
id = msg.node_id
self.clients[id] = SlaveNode(id)
logger.info("Client %r reported as ready. Currently %i clients ready to swarm." % (id, len(self.clients.ready)))
## emit a warning if the slave's clock seem to be out of sync with our clock
#if abs(time() - msg.data["time"]) > 5.0:
# warnings.warn("The slave node's clock seem to be out of sync. For the statistics to be correct the different locust servers need to have synchronized clocks.")
elif msg.type == "client_stopped":
del self.clients[msg.node_id]
if len(self.clients.hatching + self.clients.running) == 0:
self.state = STATE_STOPPED
logger.info("Removing %s client from running clients" % (msg.node_id))
elif msg.type == "stats":
events.slave_report.fire(msg.node_id, msg.data)
elif msg.type == "hatching":
self.clients[msg.node_id].state = STATE_HATCHING
elif msg.type == "hatch_complete":
self.clients[msg.node_id].state = STATE_RUNNING
self.clients[msg.node_id].user_count = msg.data["count"]
if len(self.clients.hatching) == 0:
count = sum(c.user_count for c in self.clients.itervalues())
events.hatch_complete.fire(count)
elif msg.type == "quit":
if msg.node_id in self.clients:
del self.clients[msg.node_id]
logger.info("Client %r quit. Currently %i clients connected." % (msg.node_id, len(self.clients.ready)))
elif msg.type == "exception":
self.log_exception(msg.node_id, msg.data["msg"], msg.data["traceback"])
@property
def slave_count(self):
return len(self.clients.ready) + len(self.clients.hatching) + len(self.clients.running)
class MasterLocustRunner(DistributedLocustRunner):
def __init__(self, *args, **kwargs):
super(MasterLocustRunner, self).__init__(*args, **kwargs)
class SlaveNodesDict(dict):
def get_by_state(self, state):
return [c for c in self.itervalues() if c.state == state]
@property
def ready(self):
return self.get_by_state(STATE_INIT)
@property
def hatching(self):
return self.get_by_state(STATE_HATCHING)
@property
def running(self):
return self.get_by_state(STATE_RUNNING)
self.clients = SlaveNodesDict()
self.client_stats = {}
self.client_errors = {}
self._request_stats = {}
self.server = zmqrpc.Server()
self.greenlet = Group()
self.greenlet.spawn(self.client_listener).link_exception()
# listener that gathers info on how many locust users the slaves has spawned
def on_slave_report(client_id, data):
self.clients[client_id].user_count = data["user_count"]
events.slave_report += on_slave_report
@property
def user_count(self):
return sum([c.user_count for c in self.clients.itervalues()])
def start_hatching(self, locust_count, hatch_rate):
self.num_clients = locust_count
slave_num_clients = locust_count / ((len(self.clients.ready) + len(self.clients.running)) or 1)
slave_hatch_rate = float(hatch_rate) / ((len(self.clients.ready) + len(self.clients.running)) or 1)
print "Sending hatch jobs to %i ready clients" % (len(self.clients.ready) + len(self.clients.running))
if not (len(self.clients.ready)+len(self.clients.running)):
print "WARNING: You are running in distributed mode but have no slave servers connected."
print "Please connect slaves prior to swarming."
if self.state != STATE_RUNNING and self.state != STATE_HATCHING:
RequestStats.clear_all()
for client in self.clients.itervalues():
msg = {"hatch_rate":slave_hatch_rate, "num_clients":slave_num_clients, "num_requests": self.num_requests, "host":self.host, "stop_timeout":None}
self.server.send({"type":"hatch", "data":msg})
RequestStats.global_start_time = time()
self.state = STATE_HATCHING
def stop(self):
for client in self.clients.hatching + self.clients.running:
self.server.send({"type":"stop", "data":{}})
def client_listener(self):
while True:
msg = self.server.recv()
if msg["type"] == "client_ready":
id = msg["data"]
self.clients[id] = SlaveNode(id)
print "Client %r reported as ready. Currently %i clients ready to swarm." % (id, len(self.clients.ready))
elif msg["type"] == "client_stopped":
del self.clients[msg["data"]]
if len(self.clients.hatching + self.clients.running) == 0:
self.state = STATE_STOPPED
print "Removing %s client from running clients" % (msg["data"])
elif msg["type"] == "stats":
report = msg["data"]
events.slave_report.fire(report["client_id"], report["data"])
elif msg["type"] == "hatching":
id = msg["data"]
self.clients[id].state = STATE_HATCHING
elif msg["type"] == "hatch_complete":
id = msg["data"]["client_id"]
self.clients[id].state = STATE_RUNNING
self.clients[id].user_count = msg["data"]["count"]
if len(self.clients.hatching) == 0:
count = sum(c.user_count for c in self.clients.itervalues())
events.hatch_complete.fire(count)
@property
def slave_count(self):
return len(self.clients.ready) + len(self.clients.hatching) + len(self.clients.running)
