def handle(socket, address):
log.debug('new connection from %r', address)
from uuss.server.connection import ConnectionHandler
connection_handler = ConnectionHandler(socket, address, config)
from gevent import Greenlet
fail_healthcheck_watcher = Greenlet.spawn(connection_watch_for_fail_healthcheck, connection_handler)
unset_fail_healthcheck_watcher = Greenlet.spawn(connection_watch_for_unset_fail_healthcheck, connection_handler)
shutdown_watcher = Greenlet.spawn(connection_watch_for_shutdown, connection_handler)
# make sure that if the events are set the connection_handler uses them right away
# (the greenlets watcher greenlets won't be run until after this handler has started processing)
if fail_healthcheck_event.wait(0):
connection_handler.fail_healthcheck()
if unset_fail_healthcheck_event.wait(0):
connection_handler.fail_healthcheck(False)
if shutdown_event.wait(0):
connection_handler.shutdown()
try:
connection_handler.run()
finally:
fail_healthcheck_watcher.kill()
unset_fail_healthcheck_watcher.kill()
shutdown_watcher.kill()
del connection_handler
def __init__(self, config_file='hive.cfg', key='server.key', cert='server.crt'):
self.key = key
self.cert = cert
self.config = ConfigParser.ConfigParser()
if not os.path.exists(config_file):
raise ConfigNotFound('Configuration file could not be found. ({0})'.format(config_file))
self.config.read(config_file)
#check cert and key
if not (os.path.isfile(self.key) and os.path.isfile(self.cert)):
gen_cmd = "openssl req -new -newkey rsa:1024 -days 365 -nodes -x509 -keyout server.key -out server.crt"
gen_cmd += ' && openssl rsa -in server.key -text -out server.key'
logger.error('No valid key or certificate found, '
'a selfsigned cert and key can be generated with the following '
'command: "{0}"'.format(gen_cmd))
sys.exit(1)
#inject authentication mechanism
Session.authenticator = Authenticator()
#spawning time checker
if self.config.getboolean('timecheck', 'Enabled'):
Greenlet.spawn(self.checktime)
def test_server_closess(server):
websocket = WebSocket(server_url='ws://0.0.0.0:8001')
with patch.object(websocket, 'handle_close') as mock_handle:
websocket.connect(upgrade=False)
def connection_handler():
while True:
try:
message = websocket.receive()
except Exception:
logger.exception('connection handler exploded')
raise
if message:
logger.info('got message: %s', message)
Greenlet.spawn(connection_handler)
gevent.sleep(0.01) # enough for the upgrade to happen
clients = server.clients
client_handler = list(clients.values())[0]
socket = client_handler.ws
Greenlet.spawn(socket.close)
with gevent.Timeout(1):
while mock_handle.call_count != 1:
gevent.sleep(0.01)
assert mock_handle.call_count == 1
assert mock_handle.call_args == call(close_frame=ANY)
call_param = mock_handle.call_args[1]['close_frame']
assert isinstance(call_param, Close)
def msg_received(self, msg):
# TODO: this will require reimplementing messages in some kind of
# FSM DSL, ideally programming lang agnostic
if msg.id == 1:
if not self.state is SS.NEW_CLIENT:
raise RuntimeError
self.name = msg.name
self.transport.send(factory(2))
self.state = SS.GAME_CHALLENGE
elif self.state is SS.GAME_CHALLENGE:
if msg.id == 4:
self.sock.shutdown(2)
elif msg.id == 5:
self.world = World(20, 20)
self.player = self.world.spawn(Player, owner=self)
Greenlet.spawn(self.world.ticker())
msg = factory(6)
msg.shape = self.world.shape
msg.size = self.world.size
self.transport.send(msg)
self.state = SS.GAME_IN_PROGRESS
else:
raise RuntimeError
elif self.state is SS.GAME_IN_PROGRESS:
if msg.id == 8:
args = [int(x) for x in msg.coord.split(" ")]
self.player.move(Coord(*args))
else:
raise RuntimeError
def __init__(self, work_dir, config, key='server.key', cert='server.crt', **kwargs):
"""
Main class which runs Beeswarm in Honeypot mode.
:param work_dir: Working directory (usually the current working directory)
:param config: Beeswarm configuration dictionary, None if no configuration was supplied.
:param key: Key file used for SSL enabled capabilities
:param cert: Cert file used for SSL enabled capabilities
"""
if config is None or not os.path.isdir(os.path.join(work_dir, 'data')):
Honeypot.prepare_environment(work_dir)
self.work_dir = work_dir
self.config = config
self.key = os.path.join(work_dir, key)
self.cert = os.path.join(work_dir, cert)
self._servers = []
self._server_greenlets = []
self.honeypot_id = self.config['general']['id']
Session.honeypot_id = self.honeypot_id
# write ZMQ keys to files - as expected by pyzmq
extract_keys(work_dir, config)
if not (os.path.isfile(os.path.join(work_dir, 'server.key'))):
cert_info = config['certificate_info']
if cert_info['common_name']:
cert_info['common_name'] = cert_info['common_name']
else:
cert_info['common_name'] = get_most_likely_ip()
cert, priv_key = create_self_signed_cert(cert_info['country'], cert_info['state'],
cert_info['organization'], cert_info['locality'],
cert_info['organization_unit'], cert_info['common_name'])
cert_path = os.path.join(work_dir, 'server.crt')
key_path = os.path.join(work_dir, 'server.key')
with open(cert_path, 'w') as certfile:
certfile.write(cert)
with open(key_path, 'w') as keyfile:
keyfile.write(priv_key)
send_zmq_push(SocketNames.SERVER_RELAY.value,
'{0} {1} {2}'.format(Messages.KEY.value, self.honeypot_id, priv_key))
send_zmq_push(SocketNames.SERVER_RELAY.value,
'{0} {1} {2}'.format(Messages.CERT.value, self.honeypot_id, cert))
if self.config['general']['fetch_ip']:
try:
url = 'http://api.externalip.net/ip'
req = requests.get(url)
self.honeypot_ip = req.text
logger.info('Fetched {0} as external ip for Honeypot.'.format(self.honeypot_ip))
except (Timeout, ConnectionError) as e:
logger.warning('Could not fetch public ip: {0}'.format(e))
else:
self.honeypot_ip = ''
# spawning time checker
if self.config['timecheck']['enabled']:
Greenlet.spawn(self.check_time)
def handle(sock, addr):
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
deviceId, command, params = unpack.read(sock)
try:
Device.objects(deviceId=deviceId).get()
except:
sock.send("register first!")
return None
"""
短连接命令执行
"""
if command in ComandList:
ComandList[command](sock, deviceId, params)
return None
# 建立长连接
elif command == "conn":
dc = DeviceConnect(sock, addr)
if not deviceId in DeviceList:
dt = DeviceTerminal(deviceId)
a = Greenlet.spawn(timeLiving, dt)
a.start()
DeviceList[deviceId] = dt
else:
dt = DeviceList[deviceId]
if dt.conn:
dt.conn.stop()
dt.connect(dc)
# 循环发送队列消息
a = Greenlet.spawn(DeviceSocketSend, dt, dc)
a.start()
def reset_normalized(self):
"""Deletes all normalized data from the datastore."""
logger.info("Initiating database reset - all normalized data will be deleted. (Starting timer)")
start = time.time()
for collection in self.db.collection_names():
if collection not in ["system.indexes", "hpfeed", "hpfeeds"]:
logger.warning("Dropping collection: {0}.".format(collection))
self.db.drop_collection(collection)
logger.info("All collections dropped. (Elapse: {0})".format(time.time() - start))
logger.info("Dropping indexes before bulk operation.")
self.db.hpfeed.drop_indexes()
logger.info("Indexes dropped(Elapse: {0}).".format(time.time() - start))
logger.info("Resetting normalization flags from hpfeeds collection.")
self.db.hpfeed.update(
{}, {"$set": {"normalized": False}, "$unset": {"last_error": 1, "last_error_timestamp": 1}}, multi=True
)
logger.info("Done normalization flags from hpfeeds collection.(Elapse: {0}).".format(time.time() - start))
logger.info("Recreating indexes.")
self.ensure_index()
logger.info("Done recreating indexes (Elapse: {0})".format(time.time() - start))
logger.info("Full reset done in {0} seconds".format(time.time() - start))
# This is a one-off job to generate stats for hpfeeds which takes a while.
Greenlet.spawn(self.rg.do_legacy_hpfeeds)
def main():
global session
session = Session()
session_greenlet = Greenlet.spawn(session.start)
#spam_greenlet = Greenlet.spawn(spam, session)
http_server = HTTPServer(("", 8081), SimpleHTTPRequestHandler)
http_greenlet = Greenlet.spawn(http_server.serve_forever)
WSGIServer(("", 8000), rpc_server, handler_class=WebSocketHandler).serve_forever()
def tracking(response, *args, **kwargs):
utm_url = track_page_view(request)
environ = {
"HTTP_USER_AGENT": request.environ.get("HTTP_USER_AGENT", "unknown"),
"HTTP_ACCEPT_LANGUAGE": request.environ.get("HTTP_ACCEPT_LANGUAGE", "")
}
Greenlet.spawn(send_request_to_google_analytics, utm_url, environ)
return response
def print_dots():
"""This Greenlet prints dots to the console which is useful for making
sure that other greenlets are properly not blocking."""
def m():
while True:
sys.stdout.write("."),
sys.stdout.flush()
sleep(.02)
Greenlet.spawn(m)
def cron(callback, interval):
"""Add new cronjob.
callback -- the callable to run.
interval -- timedelta specifying when the cronjob will be called.
"""
if interval == 0:
error = "cron: your timedelta has a too small resolution (< 1s)"
raise ValueError(error)
Greenlet.spawn(_cron, callback, interval)
def _eventCallback(self, event):
print event
if event.startswith('BACKEND_MESSAGE[]:[]SYSTEM_EVENT SCHEDULER_'):
self.updateUpcoming()
if event.startswith('BACKEND_MESSAGE[]:[]SYSTEM_EVENT REC_'):
# sometimes we need to wait a second or so, for recordings to actually start.
print "Recording backend event. Triggering deferred greenlet."
Greenlet.spawn(self._deferredRecordingUpdate, self)
def __call__(self):
log.debug("TICK")
[x.ticked(False) for x in self.world.objects]
[x.tick() for x in self.world.objects]
gevent.sleep(1)
if self._run:
Greenlet.spawn(self)
def start_serving(self):
""" Starts services. """
#will contain HiveStreamServer objects
self.servers = []
self.server_greenlets = []
#will contain Session objects
self.sessions = {}
self.public_ip = self.config.get('public_ip', 'public_ip')
self.fetch_ip = self.config.getboolean('public_ip', 'fetch_public_ip')
#greenlet to consume the provided sessions
self.session_consumer = consumer.Consumer(self.sessions, public_ip=self.public_ip, fetch_public_ip=self.fetch_ip)
Greenlet.spawn(self.session_consumer.start)
#protocol handlers
for c in handlerbase.HandlerBase.__subclasses__():
cap_name = 'cap_' + c.__name__
if not self.config.has_section(cap_name):
logger.warning(
"Not loading {0} capability because it has no option in configuration file.".format(c.__name__))
continue
#skip loading if disabled
if not self.config.getboolean(cap_name, 'Enabled'):
continue
port = self.config.getint(cap_name, 'port')
#carve out the options for this specific service
options = list2dict(self.config.items(cap_name))
cap = c(self.sessions, options)
try:
socket = create_socket(('0.0.0.0', port))
#Convention: All capability names which end in 's' will be wrapped in ssl.
if cap_name.endswith('s'):
server = HiveStreamServer(socket, cap.handle_session,
keyfile=self.key, certfile=self.cert)
else:
server = HiveStreamServer(socket, cap.handle_session)
self.servers.append(server)
server_greenlet = Greenlet(server.start())
self.server_greenlets.append(server_greenlet)
except _socket.error as ex:
logger.error("Could not start {0} server on port {1}. Error: {2}".format(c.__name__ , port, ex))
else:
logger.info('Started {0} capability listening on port {1}'.format(c.__name__, port))
drop_privileges()
logger.info("Hive running - see log file (hive.log) for attack events.")
gevent.joinall(self.server_greenlets)
def run(self):
signal.signal(signal.SIGINT, self.signal_handler)
self.pool = Pool(10000)
Greenlet.spawn(Server.update_all, self)
self.server = StreamServer((self.host, self.port), self.handle, spawn=self.pool)
self.logger.info('Starting FyIRCd on {0}:{1}'.format(self.host, self.port))
self.server.serve_forever()
def main():
global session
session = Session()
session_greenlet = Greenlet.spawn(session.start)
#spam_greenlet = Greenlet.spawn(spam, session)
http_server = HTTPServer(("", 8081), SimpleHTTPRequestHandler)
http_greenlet = Greenlet.spawn(http_server.serve_forever)
try:
sockjs = SockJSServer(("", 8000), router, trace=True)
sockjs.serve_forever()
except KeyboardInterrupt:
sockjs.kill()
def process_message(channel, method, properties, body):
logging.debug("Received message: %s", body)
try:
message = json.loads(body)
except (TypeError, ValueError):
logging.error("Not JSON-encoded message received: %s", body)
try:
Greenlet.spawn(handler.dispatch, message)
except Exception:
logging.exception("Unhandled exception: \n Message = %s\n\n", body)
channel.basic_ack(delivery_tag = method.delivery_tag)
def __init__(self, config="hive.cfg"):
conf_parser = ConfigParser()
conf_parser.read(config)
self.host = conf_parser.get("log_hpfeed", "host")
self.port = conf_parser.getint("log_hpfeed", "port")
self.secret = conf_parser.get("log_hpfeed", "secret").encode('latin1')
self.chan = conf_parser.get("log_hpfeed", "chan").encode('latin1')
self.ident = conf_parser.get("log_hpfeed", "ident").encode('latin1').strip()
self.enabled = True
#Used for authentication and handling of socket errors
Greenlet.spawn(self._start)
def _loop(self):
# Allocate a pool of 100 channels for playing games.
self.channels = queue.Queue()
for i in range(0, CHANNELS):
self.channels.put(i)
self.upcoming = queue.Queue()
while True:
candidates, result = self.upcoming.get()
if not candidates or not result:
break
index = self.channels.get()
Greenlet.spawn(self._play, index, candidates, result)
def start(self):
""" Starts services. """
self.servers = []
self.server_greenlets = []
#will contain Session objects
self.sessions = {}
#greenlet to consume the provided sessions
self.session_consumer = Consumer(self.sessions, self.honeypot_ip, self.config, self.status, self.work_dir)
Greenlet.spawn(self.session_consumer.start)
#protocol handlers
for c in handlerbase.HandlerBase.__subclasses__():
cap_name = c.__name__.lower()
if cap_name not in self.config['capabilities']:
logger.warning(
"Not loading {0} capability because it has no option in configuration file.".format(c.__name__))
continue
#skip loading if disabled
if not self.config['capabilities'][cap_name]['enabled']:
continue
port = self.config['capabilities'][cap_name]['port']
#carve out the options for this specific service
options = self.config['capabilities'][cap_name]
cap = c(self.sessions, options, self.users, self.work_dir)
try:
#Convention: All capability names which end in 's' will be wrapped in ssl.
if cap_name.endswith('s'):
server = StreamServer(('0.0.0.0', port), cap.handle_session,
keyfile=self.key, certfile=self.cert)
else:
server = StreamServer(('0.0.0.0', port), cap.handle_session)
self.servers.append(server)
self.status['enabled_capabilities'].append(cap_name)
server_greenlet = Greenlet(server.start())
self.server_greenlets.append(server_greenlet)
except _socket.error as ex:
logger.error("Could not start {0} server on port {1}. Error: {2}".format(c.__name__, port, ex))
else:
logger.info('Started {0} capability listening on port {1}'.format(c.__name__, port))
logger.info("Honeypot running - see log file (honeypot.log) for attack events.")
gevent.joinall(self.server_greenlets)
def run(self):
'''Start running the server'''
self.app.route('/<path:path>')(self.handle)
if self.mode == 'fork':
logger.info('Forking server with %s' % self.server)
self.pid = os.fork()
if self.pid == 0:
# I'm the child process
self._run()
exit(0)
else:
import time
time.sleep(1)
logger.info('Server started in %s' % self.pid)
elif self.mode == 'gevent':
logger.info('Launching server in greenlet %s' % self.server)
from gevent import Greenlet
self.pid = Greenlet.spawn(self._run)
# We actually need to wait a very short time before saying that
# it's started. I believe that all this does is to yield control
# from this green thread to the other green thread briefly
self.pid.join(0.01)
else:
# Blocking
self._run()
def data_scan(self, args, quiet=False, worker_count=1):
"""
Invoke cephfs-data-scan with the passed arguments, and return its stdout
:param worker_count: if greater than 1, multiple workers will be run
in parallel and the return value will be None
"""
workers = []
for n in range(0, worker_count):
if worker_count > 1:
# data-scan args first token is a command, followed by args to it.
# insert worker arguments after the command.
cmd = args[0]
worker_args = [cmd] + ["--worker_n", n.__str__(), "--worker_m", worker_count.__str__()] + args[1:]
else:
worker_args = args
workers.append(Greenlet.spawn(lambda wargs=worker_args:
self._run_tool("cephfs-data-scan", wargs, None, quiet)))
for w in workers:
w.get()
if worker_count == 1:
return workers[0].value
else:
return None
def __init__(self, cappedCol, fields=None, startFrom=True, channel=None, func=None,
autoJoin=True, retryOnDeadCursor=False, query=None):
if not cappedCol.options().get('capped'):
raise ValueError("collection_is_not_capped")
return
#super(SubToCapped, self).__init__(channel if channel else cappedCol.full_name)
#Greenlet.__init__(self)
if func:
self.onDoc = func
self.collection = cappedCol
self.isOpLog = cappedCol.full_name == 'local.oplog.rs'
self.query = query
#self.query = None if (isOpLog and startFrom) else query #@note: not compatible with
#_QUERY_OPTIONS['oplog_replay'] see:database error: no ts field in query
self.fields = fields
self.retryOnDeadCursor = retryOnDeadCursor
self.startFrom = startFrom
self.channel = channel if channel else cappedCol.full_name
self.id_start = None
self.id_last = None
self.t_start = datetime.utcnow()
self.dt_last = None
self.docs_fetched = 0
self.docs_skipped = 0
self.__stop = False
#self.start()
self.glet = Greenlet.spawn(self._run)
if autoJoin:
self.join()
def start_server(self, host, port, linked_servers=[]):
result = ObjectoPlex((host, port),
middlewares=[
PingPongMiddleware(),
LegacySubscriptionMiddleware(),
StatisticsMiddleware(),
ChecksumMiddleware(),
RoutingMiddleware(),
],
linked_servers=linked_servers)
gevent.signal(signal.SIGTERM, result.stop)
gevent.signal(signal.SIGINT, result.stop)
Greenlet.spawn(result.serve_forever)
sleep(0.1)
return result
def start(self):
""" Put job in waiting queue, it will start when dependencies are
fulfilled and resources are available"""
if self.state == self.STATE_UNDEF and self.orphan():
logging.debug("{0} is short circuiting ({1})".format(
self.name, self.state
)
)
self.events[self.STATE_RUNNING].set()
self.events[self.STATE_SUCCESSFULL].set()
return True
# Using STATE_SUCCESSFULL instead of is_successfull because we don't want to skip starting undef jobs
elif self.state == self.STATE_SUCCESSFULL:
return False
Greenlet.spawn(self._run)
return True
def _proc_loop(self, pipe, num_greenlets):
result_to_id = {}
executor = GreenletExecutor(num_greenlets=num_greenlets)
# Event to indicate that there are some tasks in flight and so write_loop
# should start waiting on their results.
tasks_in_flight = Event()
def on_kill(signum, frame):
pipe.put(None)
return
signal.signal(signal.SIGINT, on_kill) # When server is terminated.
def signal_tasks_in_flight():
if not tasks_in_flight.is_set():
tasks_in_flight.set()
def read_loop():
"""
Reads all tasks sent across the pipe by the parent process and passes
them to the local GreenletExecutor instance.
"""
while True:
task_args = pipe.get()
if task_args is None:
executor.shutdown()
signal_tasks_in_flight()
return
task = Task(*task_args)
result_to_id[task.result] = task.id
executor._submit(task)
signal_tasks_in_flight()
read_greenlet = Greenlet.spawn(read_loop)
def write_loop():
"""
Waits for results of tasks being executed by the local GreenletExecutor
and pipes the result back to the parent (callee) process.
"""
while True:
if not result_to_id:
# Wait for some result to be ready.
tasks_in_flight.clear()
try:
if read_greenlet.ready():
raise LoopExit
tasks_in_flight.wait()
continue
except LoopExit, e:
pipe.put(None)
return
ready_results = wait(objects=result_to_id.keys(), count=1)
for result in ready_results:
try:
value = result.get()
except Exception, e:
value = e
pipe.put((result_to_id[result], result.successful(), value))
del result_to_id[result]
def __init__(self, broker):
if self.CHANNEL_NAME == None:
raise NotImplementedError('Class %s does not set the CHANNEL_NAME to use.' % self.__class__.__name__)
self.broker = broker
self.broker.initChannel(self.CHANNEL_NAME)
self.greenlet = Greenlet.spawn(self._handler, self)
def __init__(self, concurrent = 2, minSecondsBetweenRequests = 0.15, defaultTimeout = None, retryStrategy = Strict(), responsePreprocessor = ResponsePreprocessor()):
if not isinstance(retryStrategy, RetryStrategy):
raise TypeError('retryStrategy must be an instance of RetryStrategy, not %s' % type(retryStrategy))
if not isinstance(responsePreprocessor, ResponsePreprocessor):
raise TypeError('responsePreprocessor must be an instance of ResponsePreprocessor, not %s' % type(responsePreprocessor))
self.session = Session()
self.pool = Pool(concurrent)
self.minSecondsBetweenRequests = minSecondsBetweenRequests
self.retryStrategy = retryStrategy
self.responsePreprocessor = responsePreprocessor
self._adapter = _DefaultTimeoutHTTPAdapter()
self.session.mount('http://', self._adapter)
self.session.mount('https://', self._adapter)
self._adapter.defaultTimeout = defaultTimeout
self._requestGroups = 0
self._requestAdded = Event()
self._requestQueue = _RequestQueue()
self._retryQueue = _RetryQueue()
self._killed = False
Requests._runningRequests.add(Greenlet.spawn(self._run))
def socketio(request):
"""The socket.io view."""
io = request.environ['socketio']
redis_sub = redis_client().pubsub()
user = username(request.user)
# Subscribe to incoming pubsub messages from redis.
def subscriber(io):
redis_sub.subscribe(room_channel())
redis_client().publish(room_channel(), user + ' connected.')
while io.connected():
for message in redis_sub.listen():
if message['type'] == 'message':
io.send(message['data'])
greenlet = Greenlet.spawn(subscriber, io)
# Listen to incoming messages from client.
while io.connected():
message = io.recv()
if message:
redis_client().publish(room_channel(), user + ': ' + message[0])
# Disconnected. Publish disconnect message and kill subscriber greenlet.
redis_client().publish(room_channel(), user + ' disconnected')
greenlet.throw(Greenlet.GreenletExit)
return HttpResponse()
def parent(my_id):
print "parent start", my_id
time.sleep(random.random())
greenlets = []
for i in xrange(NUM_CHILDREN):
greenlets.append(Greenlet.spawn(child, my_id, i))
for greenlet in greenlets: greenlet.join()
print "parent end", my_id
def api_backend(raiden_network, rest_api_port_number):
raiden_api = RaidenAPI(raiden_network[0].raiden)
rest_api = RestAPI(raiden_api)
api_server = APIServer(rest_api)
api_server.flask_app.config['SERVER_NAME'] = 'localhost:{}'.format(
rest_api_port_number)
# TODO: Find out why tests fail with debug=True
server = Greenlet.spawn(
api_server.run,
port=rest_api_port_number,
debug=False,
use_evalex=False,
)
# Fixes flaky test, were requests are done prior to the server initializing
# the listening socket.
# https://github.com/raiden-network/raiden/issues/389#issuecomment-305551563
wait_for_listening_port(rest_api_port_number)
yield api_server, rest_api
server.kill(block=True, timeout=10)
def api_backend(rest_api_port_number):
# Initializing it without raiden_service.api here since that is a
# function scope fixture. We will inject it to rest_api object later
rest_api = RestAPI(None)
api_server = APIServer(rest_api)
api_server.flask_app.config['SERVER_NAME'] = 'localhost:{}'.format(rest_api_port_number)
# TODO: Find out why tests fail with debug=True
server = Greenlet.spawn(
api_server.run,
rest_api_port_number,
debug=False,
use_evalex=False,
)
# Fixes flaky test, were requests are done prior to the server initializing
# the listening socket.
# https://github.com/raiden-network/raiden/issues/389#issuecomment-305551563
wait_for_listening_port(rest_api_port_number)
yield api_server, rest_api
server.kill(block=True, timeout=10)
def spanning_threads():
def foo(message, n):
"""
Each thread will be passed the message, and n arguments
in its initialization.
"""
gevent.sleep(n)
print(message)
# Initialize a new Greenlet instance running the named function
# foo
thread1 = Greenlet.spawn(foo, "Hello", 1)
# Wrapper for creating and running a new Greenlet from the named
# function foo, with the passed arguments
thread2 = gevent.spawn(foo, "I live!", 2)
# Lambda expressions
thread3 = gevent.spawn(lambda x: (x + 1), 2)
threads = [thread1, thread2, thread3]
# Block until all threads complete.
gevent.joinall(threads)
def __init__(self,
cappedCol,
fields=None,
startFrom=True,
channel=None,
func=None,
autoJoin=True,
retryOnDeadCursor=False,
query=None):
if not cappedCol.options().get('capped'):
raise ValueError("collection_is_not_capped")
return
#super(SubToCapped, self).__init__(channel if channel else cappedCol.full_name)
#Greenlet.__init__(self)
if func:
self.onDoc = func
self.collection = cappedCol
self.isOpLog = cappedCol.full_name == 'local.oplog.rs'
self.query = query
#self.query = None if (isOpLog and startFrom) else query #@note: not compatible with
#_QUERY_OPTIONS['oplog_replay'] see:database error: no ts field in query
self.fields = fields
self.retryOnDeadCursor = retryOnDeadCursor
self.startFrom = startFrom
self.channel = channel if channel else cappedCol.full_name
self.id_start = None
self.id_last = None
self.t_start = datetime.utcnow()
self.dt_last = None
self.docs_fetched = 0
self.docs_skipped = 0
self.__stop = False
#self.start()
self.glet = Greenlet.spawn(self._run)
if autoJoin:
self.join()
def __init__(self,
work_dir,
config,
key='server.key',
cert='server.crt',
**kwargs):
"""
Main class which runs Beeswarm in Honeypot mode.
:param work_dir: Working directory (usually the current working directory)
:param config: Beeswarm configuration dictionary, None if no configuration was supplied.
:param key: Key file used for SSL enabled capabilities
:param cert: Cert file used for SSL enabled capabilities
"""
if fs.__version__ != '0.5.4':
os.exit('the python fs package must be verison 0.5.4')
if config is None or not os.path.isdir(os.path.join(work_dir, 'data')):
Honeypot.prepare_environment(work_dir)
self.work_dir = work_dir
self.config = config
self.key = os.path.join(work_dir, key)
self.cert = os.path.join(work_dir, cert)
self._servers = []
self._server_greenlets = []
self.honeypot_id = self.config['general']['id']
Session.honeypot_id = self.honeypot_id
# write ZMQ keys to files - as expected by pyzmq
extract_keys(work_dir, config)
if not (os.path.isfile(os.path.join(work_dir, 'server.key'))):
cert_info = config['certificate_info']
if cert_info['common_name']:
cert_info['common_name'] = cert_info['common_name']
else:
cert_info['common_name'] = get_most_likely_ip()
cert, priv_key = create_self_signed_cert(
cert_info['country'], cert_info['state'],
cert_info['organization'], cert_info['locality'],
cert_info['organization_unit'], cert_info['common_name'])
cert_path = os.path.join(work_dir, 'server.crt')
key_path = os.path.join(work_dir, 'server.key')
with open(cert_path, 'w') as certfile:
certfile.write(cert)
with open(key_path, 'w') as keyfile:
keyfile.write(priv_key)
send_zmq_push(
SocketNames.SERVER_RELAY.value,
'{0} {1} {2}'.format(Messages.KEY.value, self.honeypot_id,
priv_key))
send_zmq_push(
SocketNames.SERVER_RELAY.value,
'{0} {1} {2}'.format(Messages.CERT.value, self.honeypot_id,
cert))
if self.config['general']['fetch_ip']:
try:
url = 'http://api.externalip.net/ip'
req = requests.get(url)
self.honeypot_ip = req.text
logger.info('Fetched {0} as external ip for Honeypot.'.format(
self.honeypot_ip))
except (Timeout, ConnectionError) as e:
logger.warning('Could not fetch public ip: {0}'.format(e))
else:
self.honeypot_ip = ''
# spawning time checker
if self.config['timecheck']['enabled']:
Greenlet.spawn(self.check_time)
def handle(environ, start_response):
start_response('200 OK', [('Content-Type', 'text/plain')])
body = queue.Queue()
Greenlet.spawn(current_time, body)
return body
def run(self):
def on_text(payload):
self.outq.put(XboxEvent(EVT_XBOX_SYSTEMTEXT_INPUT_ID, payload))
def on_connect_request(req: ConnectionRequest):
auth_mgr = AuthenticationManager.from_file(TOKENS_FILE)
auth_mgr.dump(TOKENS_FILE)
userhash = auth_mgr.userinfo.userhash
token = auth_mgr.xsts_token.jwt
for c in Console.discovered():
if str(c.uuid) == str(req.console_dict['uuid']):
self._console = c
if self._console is None:
self.outq.put(
XboxEvent(EVT_XBOX_DISCONNECT_ID, "Failed to connect"))
return
self._console.add_manager(TextManager)
self._console.text.on_systemtext_input += on_text
self._console.protocol.on_timeout += lambda: self._timedout.set()
try:
status = self._console.connect(userhash, token)
except OSError as e:
self.outq.put(
XboxEvent(EVT_XBOX_DISCONNECT_ID,
f"Failed to connect {e}"))
return
self._console.wait(1)
self.outq.put(XboxEvent(EVT_XBOX_CONNECT_ID,
self._console.address))
def mainloop(skip_connection=False):
while True:
if skip_connection:
new_item = None
try:
new_item = self.inq.get_nowait()
except:
pass
if new_item is not None:
if isinstance(new_item, DiscoverRequest):
try:
discovered = Console.discover(
addr=new_item.addr)
except OSError as e:
self.outq.put(
XboxEvent(EVT_XBOX_DISCOVERYFAILURE_ID, e))
for console in discovered:
dc = DiscoveredConsole(console)
self.outq.put(
XboxEvent(EVT_XBOX_DISCOVERED_CONSOLE_ID,
dc))
if len(discovered) < 1:
try:
Console.__protocol__.start()
Console.__protocol__._discover(
xbox.sg.factory.discovery(),
xbox.sg.protocol.BROADCAST, 5)
except OSError as e:
self.outq.put(
XboxEvent(EVT_XBOX_DISCOVERYFAILURE_ID,
e))
self.inq.task_done()
if isinstance(new_item,
ConnectionRequest) or isinstance(
new_item, DiscoveredConsole):
on_connect_request(
new_item) # this instantiates self._console
self.inq.task_done()
if isinstance(new_item, SystemTextSend):
if self._console is None or not self._console.connected:
error = "Failed to send text. Disconnected!"
self.outq.put(error)
print(error)
else:
self._console.send_systemtext_input(
new_item.text)
self._console.finish_text_input()
self.outq.put(
f"Sent {new_item.text} to console")
self.inq.task_done()
if isinstance(new_item, DisconnectRequest):
if self._console:
if self._console.connected:
self._console.protocol._stop_event.set()
self._console.disconnect()
self.inq.task_done()
self.outq.put(
XboxEvent(
EVT_XBOX_DISCONNECT_ID,
self._console.address
if self._console else None))
else:
self.inq.task_done()
self.outq.put(
XboxEvent(
EVT_XBOX_DISCONNECT_ID,
self._console.address
if self._console else None))
else:
self.inq.task_done()
self.outq.put(
XboxEvent(
EVT_XBOX_DISCONNECT_ID,
self._console.address
if self._console else None))
if new_item == POISON_PILL:
if self._console:
if self._console.connected:
self._console.disconnect()
self._console = None
self.inq.put("STOP")
self.inq.task_done()
break
gevent.sleep(0)
if self._console:
if self._console.connected:
# print("console instantiated. started:", self._console.protocol.started)
# print("console status:", self._console.connection_state)
# print("Closed?:", self._console.protocol.closed)
# print("Timedout?:", self._timedout.is_set())
try:
""
gevent.sleep(
self._console.protocol.HEARTBEAT_INTERVAL)
self._console.protocol.ack([], [],
ServiceChannel.Core,
need_ack=True)
except (OSError, AttributeError) as e:
self._console.protocol.on_timeout()
self.outq.put(
XboxEvent(EVT_XBOX_DISCONNECT_ID,
f"Failed to connect {e}"))
finally:
if self._stay_connected.is_set(
) and self._timedout.is_set():
gevent.sleep(10)
on_connect_request(
ConnectionRequest(self._console))
gevent.sleep(0.1)
event_thread = Greenlet.spawn(mainloop, True)
event_thread.start()
event_thread.join()
def start_client(self, cb):
def start():
cb(self.start_client_sync())
return Greenlet.spawn(start)
def run_stress_worker(server_host_ip, server_port, num_runs, worker_model_real,
chunk_str):
"""
Run the workers loop for the basic stress test. This involves
creating a a set of workers according to the keys in STRESS_KEYS_FOLDER then:
- registering them with the server
- get the current global model
- send an update
- Get the next global model.
Parameters
----------
server_host_ip: str
The ip-address of the host of the server.
server_port: int
The port at which the serer should listen to
num_runs: int
Number of runs of the sending of models etc. to perform
worker_model_real: bool
If true, the global model returned is a bianry serialized version of
MobileNetV2 that is used in the plantvillage example.
chunk_str: str
String giving the chunk of keys to use.
"""
workers = []
if worker_model_real:
model_data = io.BytesIO()
torch.save(models.mobilenet_v2(pretrained=True), model_data)
bin_model = model_data.getvalue()
else:
bin_model = msgpack.packb("A 'local model update'!!")
for fn in get_worker_keys_from_chunk(chunk_str):
workers.append(
SimpleLPWorker(server_host_ip, server_port,
os.path.join(STRESS_KEYS_FOLDER, fn)))
num_workers = len(workers)
for i, worker in enumerate(workers):
logger.info(f'Registering {i} th worker')
worker.worker.register_worker()
# get the current global model and check
for i, worker in enumerate(workers):
print(
f"Requesting global model for {worker.worker.worker_id} (no. {i}) "
)
worker.global_model_changed_callback(worker.worker.get_global_model())
done_count = 0
def run_wg(gl_worker, num):
nonlocal done_count
logger.info(
f"Starting long poll for {gl_worker.worker.worker_id} (no. {num})")
gl_worker.global_model_changed_callback(
gl_worker.worker.get_global_model())
logger.info(f"Long poll for {gl_worker.worker.worker_id} finished")
done_count += 1
try:
for run_no in range(num_runs):
logger.info(f"********************** STARTING RUN {run_no + 1}:")
sleep(5)
for i, worker in enumerate(workers):
response = worker.worker.send_model_update(bin_model)
logger.info(
f"Response from server sending model update: {response}")
logger.info(f"Spawning for worker {i}")
Greenlet.spawn(run_wg, worker, i)
if (i + 1) % 10 == 0:
sleep(0.5)
while done_count < num_workers:
sleep(1)
logger.info(
f"{done_count} workers have received the global model update - need to get to {num_workers}..."
)
done_count = 0
except Exception as e:
print(e)
exit()
def _proc_loop(self, pipe, num_greenlets):
result_to_id = {}
executor = GreenletExecutor(num_greenlets=num_greenlets)
# Event to indicate that there are some tasks in flight and so write_loop
# should start waiting on their results.
tasks_in_flight = Event()
def on_kill(signum, frame):
pipe.put(None)
return
signal.signal(signal.SIGINT, on_kill) # When server is terminated.
def signal_tasks_in_flight():
if not tasks_in_flight.is_set():
tasks_in_flight.set()
def read_loop():
"""
Reads all tasks sent across the pipe by the parent process and passes
them to the local GreenletExecutor instance.
"""
while True:
task_args = pipe.get()
if task_args is None:
executor.shutdown()
signal_tasks_in_flight()
return
task = Task(*task_args)
result_to_id[task.result] = task.id
executor._submit(task)
signal_tasks_in_flight()
read_greenlet = Greenlet.spawn(read_loop)
def write_loop():
"""
Waits for results of tasks being executed by the local GreenletExecutor
and pipes the result back to the parent (callee) process.
"""
while True:
if not result_to_id:
# Wait for some result to be ready.
tasks_in_flight.clear()
try:
if read_greenlet.ready():
raise LoopExit
tasks_in_flight.wait()
continue
except LoopExit, e:
pipe.put(None)
return
ready_results = wait(objects=result_to_id.keys(), count=1)
for result in ready_results:
try:
value = result.get()
except Exception, e:
value = e
pipe.put(
(result_to_id[result], result.successful(), value))
del result_to_id[result]
def __init__(self,
inputs,
outputs,
zmq_args=None,
datastore="ait.core.db.InfluxDBBackend",
**kwargs):
"""
Params:
inputs: names of inbound streams plugin receives data from
outputs: names of outbound streams plugin sends its data to
zmq_args: dict containing the follow keys:
zmq_context
zmq_proxy_xsub_url
zmq_proxy_xpub_url
Defaults to empty dict. Default values
assigned during instantiation of parent class.
datastore: path to database backend to use
**kwargs: (optional) Dependent on requirements of child class.
"""
super(AITOpenMctPlugin, self).__init__(inputs, outputs, zmq_args,
**kwargs)
log.info("Running AIT OpenMCT Plugin")
self._datastore = datastore
# Initialize state fields
# Debug state fields
self._debugEnabled = AITOpenMctPlugin.DEFAULT_DEBUG
self._debugMimicRepeat = False
# Port value for the server
self._servicePort = AITOpenMctPlugin.DEFAULT_PORT
# Flag indicating if we should create a database connection for historical queries
self._databaseEnabled = AITOpenMctPlugin.DEFAULT_DATABASE_ENABLED
# Check for AIT config overrides
self._check_config()
# Setup server state
self._app = bottle.Bottle()
self._servers = []
# Queues for AIT events events
self._tlmQueue = api.GeventDeque(maxlen=100)
# Load AIT tlm dict and create OpenMCT format of it
self._aitTlmDict = tlm.getDefaultDict()
self._mctTlmDict = DictUtils.format_tlmdict_for_openmct(
self._aitTlmDict)
# Create lookup from packet-uid to packet def
self._uidToPktDefMap = DictUtils.create_uid_pkt_map(self._aitTlmDict)
# Attempt to initialize database, None if no DB
self._database = self.load_database(**kwargs)
# Maintains a set of active websocket structs
self._socket_set = set()
# Spawn greenlets to poll telemetry
self.tlm_poll_greenlet = Greenlet.spawn(
self.poll_telemetry_periodically)
gevent.spawn(self.init)
return redirect(url_for(onelove.api.endpoint('doc')))
@onelove.socketio.on('connect')
def on_connect():
from onelove.models import User
token = request.args.get('token', None)
request.namespace = '/onelove'
if token is None:
disconnect()
return
current_identity = None
try:
current_identity = onelove.jwt.jwt_decode_callback(token)
except:
disconnect()
if current_identity is None:
disconnect()
return
user = User.objects.get(id=current_identity['identity'])
join_room(str(user.id))
if os.environ.get("WERKZEUG_RUN_MAIN") == "true":
Greenlet.spawn(monitor)
if __name__ == '__main__':
manager.run()
def watching(self):
for key, callback_handler in self.__handers__.items():
Greenlet.spawn(self.watch, key, callback_handler)
#!/usr/bin/env python
from gevent import Greenlet
from brubeck.request_handling import Brubeck
from brubeck.templating import load_jinja2_env
import handlers
import os
import sys
Greenlet.spawn(handlers.PollHandler.loop, delay=5)
config = {
'mongrel2_pair': ('ipc://run/mongrel2_send', 'ipc://run/mongrel2_recv'),
'handler_tuples': [
(r'^/$', handlers.IndexHandler),
(r'^/poll$', handlers.PollHandler),
],
'template_loader':
load_jinja2_env('./templates'),
}
app = Brubeck(**config)
if __name__ == '__main__':
app.run()
def __call__(self, client, msg):
if msg.command == '001':
self.client = client
client.send_message(message.Join('#resistance'))
Greenlet.spawn(self._loop)
elif msg.command == 'PING':
client.send_message(message.Command(msg.params, 'PONG'))
elif msg.command == '353':
if msg.params[2] != '#resistance':
# When joining specific bot private channels, see if the bot is
# already there waiting and don't require rejoin.
waiting = [u.strip('+@') for u in msg.params[3:]]
for g in self.games:
for b in [b for b in g.bots if b.name in waiting]:
if b.channel == msg.params[2]:
if b._join:
b._join.set()
return
self.competitors = [u.strip('+@') for u in msg.params[3:]]
self.competitors.remove(client.nick)
elif msg.command == 'JOIN':
user = msg.prefix.split('!')[0].strip('+@')
if user == client.nick:
return
channel = msg.params[0].lstrip(':')
if channel != '#resistance':
for g in self.games:
for b in g.bots:
if b.channel == channel:
if b._join:
b._join.set()
return
if channel in b.channel:
# Bot joined shared game channel, used for global chat.
return
print("Not waiting for a player to join this channel.", file=sys.stderr)
else:
self.competitors.append(user)
elif msg.command == 'PART':
user = msg.prefix.split('!')[0].strip('+@')
if user == client.nick:
return
channel = msg.params[0].lstrip(':')
if channel == '#resistance':
self.competitors.remove(user)
return
else:
for g in self.games:
for b in g.bots:
if b.channel == channel and b._part:
# Only leave the channel once the other has left, to avoid
# synchronization problems when batch processing games.
b._part.set()
return
elif msg.command == 'PRIVMSG':
# Any human may ask this server to run games with available players.
channel = msg.params[0].lstrip(':')
if channel == '#resistance':
if msg.params[1].lower() == 'play':
self.run(' '.join(msg.params[2:]))
return
# Connecting bots always self-identify as bot for future reference.
if len(msg.params) > 1 and msg.params[1] == 'BOT':
self.identities.append(msg.prefix.split('!')[0])
for g in self.games:
user = msg.prefix.split('!')[0].strip('+@')
if g.channel == channel:
g.file.write('[%s] ' % user + ' '.join(msg.params[1:])+'\n')
g.file.flush()
# Check if this is a report message about sabotages in
# games played between humans alone or with bots.
if g.channel == channel and msg.params[1].upper() == 'SABOTAGES':
try:
remaining = int(msg.params[2].strip('.,!;'))
except ValueError:
return
for bot in g.bots:
if bot._sabotage is not None:
r = bool(remaining > 0)
bot.send("SABOTAGED %s" % showYesOrNo(r))
if bot.spy:
bot._sabotage.set(r)
remaining -= 1
else:
bot._sabotage.set(False)
bot._sabotage = None
if g.channel == channel and msg.params[1].upper() == 'VOTES':
votes = [parseYesOrNo(v) for v in msg.params[2:]]
for bot in g.bots:
if bot._vote is not None and bot.name not in self.identities:
v = votes.pop(0)
bot.send("VOTED %s." % showYesOrNo(v))
bot._vote.set(v)
if g.channel == channel and msg.params[1].upper() == 'SELECTS':
for bot in g.bots:
if bot._select is not None:
bot.send("SELECTED %s" % ' '.join(msg.params[2:]))
bot.process_SELECTED(msg.params)
# Now check if a bot is expecting a message, and pass it along.
for bot in g.bots:
if bot.channel != channel:
continue
name = 'process_'+msg.params[1].upper()
if hasattr(self, name):
process = getattr(self, name)
process(msg.params)
elif hasattr(bot, name):
process = getattr(bot, name)
process(msg.params)
elif bot.expecting:
try:
bot.expecting(msg.params)
except:
# Comments can overflow in multiple lines.
pass
def main():
threads = []
for i in range(0, 10):
threads.append(Greenlet.spawn(test, i))
gevent.joinall(threads)
import gevent
from gevent import Greenlet
def foo(message, n):
"""
Each thread will be passed the message, and n arguments
in its initialization.
"""
print(message)
gevent.sleep(n)
print(message, "complete...")
# Initialize a new Greenlet instance running the named function
# foo
thread1 = Greenlet.spawn(foo, "Hello", 1)
# Wrapper for creating and running a new Greenlet from the named
# function foo, with the passed arguments
thread2 = gevent.spawn(foo, "I live!", 2)
# Lambda expressions
thread3 = gevent.spawn(lambda x: (x+1), 2)
threads = [thread1, thread2, thread3]
# Block until all threads complete.
gevent.joinall(threads)
def start(self):
self.stop()
self.loop = Greenlet.spawn(self.run)
self.loop.start()
def __gblocking__(fun, *args, **kwargs):
return Greenlet.spawn(fun, *args, **kwargs).get()
# 示例
import gevent
from gevent import Greenlet
def foo(message, n):
"""
Each thread will be passed the message, and n arguments in its initialization.
每个线程初始化时都需要传递 message 和 n 参数
"""
gevent.sleep(n)
print(message)
# Initialize a new Greenlet instance running the named function foo
# 初始化一个新的 Greenlet 实例，用来运行名为 foo 的函数
thread1 = Greenlet.spawn(foo, "Hello", 1) ### 省去了线程的 start() 步骤 ###
# Wrapper for creating and running a new Greenlet from the named function foo, with the passed arguments
# 新建一个将要运行的 Greenlet 实例，它将名为 foo 的函数和这函数需要的参数封装起来
thread2 = gevent.spawn(foo, "I live!", 2) ### 注意 thread1 和 thread2 的不同点，这里是用 gevent.spawn， 而 thread1 是用 Greenlet.spawn ###
# Lambda expressions
# Lambda 表达式的封装
thread3 = gevent.spawn(lambda x: (x+1), 2)
threads = [thread1, thread2, thread3]
# Block until all threads complete.
# 阻塞，等待所有线程都运行完为止(其实是等待所有的协程运行完毕)
gevent.joinall(threads)
def __call__(self, client, msg):
if msg.command == '001':
self.client = client
client.send_message(message.Join('#resistance'))
Greenlet.spawn(self._loop)
elif msg.command == 'PING':
client.send_message(message.Command(msg.params, 'PONG'))
elif msg.command == '353':
if msg.params[2] != '#resistance':
# When joining specific bot private channels, see if the bot is
# already there waiting and don't require rejoin.
waiting = [u.strip('+@') for u in msg.params[3:]]
for g in self.games:
for b in [b for b in g.bots if b.name in waiting]:
if b.channel == msg.params[
2] and b._join and not b._join.ready():
b._join.set()
return
self.competitors = [u.strip('+@') for u in msg.params[3:]]
self.competitors.remove(client.nick)
elif msg.command == 'JOIN':
user = msg.prefix.split('!')[0].strip('+@')
if user == client.nick:
return
channel = msg.params[0].lstrip(':')
if channel != '#resistance':
for g in self.games:
for b in g.bots:
if b.channel == channel and b._join:
b._join.set()
return
assert False, "Not waiting for a player to join this channel."
else:
self.competitors.append(user)
elif msg.command == 'PART':
user = msg.prefix.split('!')[0].strip('+@')
if user == client.nick:
return
channel = msg.params[0].lstrip(':')
if channel == '#resistance':
self.competitors.remove(user)
return
else:
for g in self.games:
for b in g.bots:
if b.channel == channel and b._part:
# Only leave the channel once the other has left, to avoid
# synchronization problems when batch processing games.
b._part.set()
return
elif msg.command == 'PRIVMSG':
channel = msg.params[0].lstrip(':')
if channel == '#resistance':
if msg.params[1] == 'PLAY':
self.run(' '.join(msg.params[2:]))
return
if msg.params[1] == 'BOT':
self.identities.append(msg.prefix.split('!')[0])
for g in self.games:
# First check if this is a report message about sabotages in
# games played between humans alone or with bots.
if g.channel == channel and msg.params[1].upper(
) == 'SABOTAGES':
remaining = int(msg.params[2].strip('.,!;'))
for bot in g.bots:
if bot._sabotage is not None:
bot.send("SABOTAGES %i" % (remaining))
if bot.spy:
bot._sabotage.set(bool(remaining > 0))
remaining -= 1
else:
bot._sabotage.set(False)
# Now check if a bot is expecting a message, and pass it along.
for bot in g.bots:
if bot.channel != channel:
continue
name = 'process_' + msg.params[1].upper()
if name == 'process_COMMENT':
pass
elif hasattr(bot, name):
process = getattr(bot, name)
process(msg.params)
elif bot.expecting:
bot.expecting(msg.params)
def __enter__(self):
if self.greenlet:
raise error.ProgrammingError("UI is already running and can't be started again")
self.greenlet = Greenlet.spawn(self.run)
return self
def test_worker_persistence():
worker_ids = []
added_workers = []
worker_updates = {}
global_model_version = "1"
worker_global_model_version = "0"
os.environ[ADMIN_USERNAME] = 'admin'
os.environ[ADMIN_PASSWORD] = 'str0ng_s3cr3t'
admin_auth = ('admin', 'str0ng_s3cr3t')
public_keys = []
private_keys = []
num_workers = 6
num_pre_load_workers = 3
worker_key_file_prefix = 'worker_key_file'
for n in range(num_workers):
private_key, public_key = gen_pair(worker_key_file_prefix + f'_{n}')
private_keys.append(
private_key.encode(encoder=HexEncoder).decode('utf-8'))
public_keys.append(
public_key.encode(encoder=HexEncoder).decode('utf-8'))
# write the pre-loaded keys to the
worker_key_file = 'worker_public_keys.txt'
with open(worker_key_file, 'w') as f:
for public_key in public_keys[0:num_pre_load_workers]:
f.write(public_key + os.linesep)
def begin_server(server, server_adapter):
server.start_server(server_adapter)
def test_register_func_cb(id):
worker_ids.append(id)
def test_unregister_func_cb(id):
worker_ids.remove(id)
def test_ret_global_model_cb():
return create_model_dict(msgpack.packb("Pickle dump of a string"),
global_model_version)
def is_global_model_most_recent(version):
return int(version) == global_model_version
def test_rec_server_update_cb(worker_id, update):
if worker_id in worker_ids:
worker_updates[worker_id] = update
return f"Update received for worker {worker_id[0:WID_LEN]}."
else:
return f"Unregistered worker {worker_id[0:WID_LEN]} tried to send an update."
def get_signed_phrase(private_key, phrase=b'test phrase'):
return SigningKey(private_key, encoder=HexEncoder).sign(phrase).hex()
if os.path.exists('workers_db.json'):
os.remove('workers_db.json')
server = DCFServer(
register_worker_callback=test_register_func_cb,
unregister_worker_callback=test_unregister_func_cb,
return_global_model_callback=test_ret_global_model_cb,
is_global_model_most_recent=is_global_model_most_recent,
receive_worker_update_callback=test_rec_server_update_cb,
server_mode_safe=True,
load_last_session_workers=True,
path_to_keys_db='workers_db.json',
key_list_file=worker_key_file)
worker_updates = {}
worker_ids = []
added_workers = []
stoppable_server = StoppableServer(host=get_host_ip(), port=8080)
server_gl = Greenlet.spawn(begin_server, server, stoppable_server)
sleep(2)
assert len(server.worker_manager.public_keys_db) == 3
# Register a set of workers using the admin API and test registration
for i in range(num_pre_load_workers, num_workers):
admin_registered_worker = {
PUBLIC_KEY_STR: public_keys[i],
REGISTRATION_STATUS_KEY: True
}
response = requests.post(
f"http://{server.server_host_ip}:{server.server_port}/{WORKERS_ROUTE}",
json=admin_registered_worker,
auth=admin_auth)
added_worker_dict = json.loads(response.content.decode('utf-8'))
idx = i - num_pre_load_workers
assert len(worker_ids) == idx + 1
assert worker_ids[idx] == added_worker_dict[WORKER_ID_KEY]
added_workers.append(added_worker_dict[WORKER_ID_KEY])
assert len(server.worker_manager.public_keys_db) == 6
for doc in server.worker_manager.public_keys_db.all():
assert doc[PUBLIC_KEY_STR] in public_keys
# Send updates and receive global updates for the registered workers
# This should succeed
worker_updates = {}
for i in range(num_pre_load_workers, num_workers):
# send updates
signed_phrase = get_signed_phrase(
private_keys[i],
hashlib.sha256(msgpack.packb("Model update!!")).digest())
response = requests.post(
f"http://{server.server_host_ip}:{server.server_port}/"
f"{RECEIVE_WORKER_UPDATE_ROUTE}/{added_workers[i - num_pre_load_workers]}",
files={
WORKER_MODEL_UPDATE_KEY:
zlib.compress(msgpack.packb("Model update!!")),
SIGNED_PHRASE:
signed_phrase
}).content
assert msgpack.unpackb(worker_updates[worker_ids[
i - num_pre_load_workers]]) == "Model update!!"
assert response.decode(
"UTF-8"
) == f"Update received for worker {added_workers[i - num_pre_load_workers][0:WID_LEN]}."
# receive updates
challenge_phrase = requests.get(
f"http://{server.server_host_ip}:{server.server_port}/"
f"{CHALLENGE_PHRASE_ROUTE}/{added_workers[i - num_pre_load_workers]}"
).content
model_return_binary = requests.post(
f"http://{server.server_host_ip}:{server.server_port}/{RETURN_GLOBAL_MODEL_ROUTE}",
json={
WORKER_ID_KEY: added_workers[i - num_pre_load_workers],
SIGNED_PHRASE: get_signed_phrase(private_keys[i],
challenge_phrase),
LAST_WORKER_MODEL_VERSION: "0"
}).content
model_return = msgpack.unpackb(zlib.decompress(model_return_binary))
assert isinstance(model_return, dict)
assert model_return[GLOBAL_MODEL_VERSION] == global_model_version
assert msgpack.unpackb(
model_return[GLOBAL_MODEL]) == "Pickle dump of a string"
stoppable_server.shutdown()
worker_ids = []
worker_updates = {}
server = DCFServer(
register_worker_callback=test_register_func_cb,
unregister_worker_callback=test_unregister_func_cb,
return_global_model_callback=test_ret_global_model_cb,
is_global_model_most_recent=is_global_model_most_recent,
receive_worker_update_callback=test_rec_server_update_cb,
server_mode_safe=True,
load_last_session_workers=True,
path_to_keys_db='workers_db.json',
key_list_file=worker_key_file)
assert len(server.worker_manager.public_keys_db) == 6
assert len(server.worker_manager.allowed_workers) == 6
for doc in server.worker_manager.public_keys_db.all():
assert doc[PUBLIC_KEY_STR] in server.worker_manager.allowed_workers
stoppable_server = StoppableServer(host=get_host_ip(), port=8080)
server_gl = Greenlet.spawn(begin_server, server, stoppable_server)
sleep(2)
# Delete existing workers and check this works.
for i in range(num_pre_load_workers):
response = requests.delete(
f"http://{server.server_host_ip}:{server.server_port}/{WORKERS_ROUTE}"
f"/{added_workers[i]}",
auth=admin_auth)
message_dict = json.loads(response.content.decode('utf-8'))
assert SUCCESS_MESSAGE_KEY in message_dict
assert len(worker_ids) == 0
assert len(server.worker_manager.public_keys_db) == 3
assert len(server.worker_manager.allowed_workers) == 3
for doc in server.worker_manager.public_keys_db.all():
assert doc[PUBLIC_KEY_STR] in server.worker_manager.allowed_workers
stoppable_server.shutdown()
# delete the files
for n in range(num_workers):
os.remove(worker_key_file_prefix + f'_{n}')
os.remove(worker_key_file_prefix + f'_{n}.pub')
os.remove(worker_key_file)
os.remove('workers_db.json')
os.remove('workers_db.json.bak')
import gevent
from gevent import Greenlet
def foo(message, n):
gevent.sleep(0)
print(message)
thread1 = Greenlet.spawn(foo, 'Hello', 1)
thread2 = gevent.spawn(foo, 'I live', 2)
thread3 = gevent.spawn(lambda x: (x + 1), 2)
threads = [thread1, thread2, thread3]
gevent.joinall(threads)
continue
def signal_handler(signum, aaa):
sys.exit()
if __name__ == "__main__":
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
threads = []
start_time = time.time()
for i in range(threadNum):
thrd = Ranker(i)
threads.append(thrd)
g = Greenlet.spawn(thrd.run)
#g.start()
#thrd.start()
while True:
time.sleep(status_interval)
print "Rankered Count: %d" % cur_rankered_count
print "Rankered succ: %d, correct count: %d rankered failed: %d, req failed: %d" % (
cur_rankered_succ, correct_count, cur_rankered_fail, req_failed)
print "Rankered QPS: %f" % (cur_rankered_succ /
(time.time() - start_time))
if cur_rankered_succ == 0:
latency = 0
recall = 0
accuracy = 0
else:
def start(self, task_list, worker_func, worker_count=4):
for i in range(worker_count):
w = Greenlet.spawn(worker_func, task_list.get_queue(), self.quit_workers)
self.workers.append(w)
def _acquire_partition_ownership(self):
# cleanup any finished greenlets
self._cleanup_greenlets()
# this variable will help us decide if we need to call callback
updated_curr_ownership = False
# list of partitions for which locks have to be released
release_lock_list = []
self._logger.info("known servers: %s" % self._con_hash.get_all_nodes())
for part in range(0, self._max_partition):
if (part in self._target_part_ownership_list):
if (part in self._curr_part_ownership_list):
# do nothing, I already have ownership of this partition
self._logger.info("No need to acquire ownership of:" +
str(part))
else:
# I need to acquire lock for this partition before I own
if (part in list(self._part_lock_task_dict.keys())):
try:
self._part_lock_task_dict[part].get(block=False)
except:
# do nothing there is already a greenlet running to
# acquire the lock
self._logger.error("Already a greenlet running to"
" acquire:" + str(part))
continue
# Greenlet died without getting ownership. Cleanup
self._logger.error("Cleanup stale greenlet running to"
" acquire:" + str(part))
del self._part_lock_task_dict[part]
self._logger.error("Starting greenlet running to"
" acquire:" + str(part))
# launch the greenlet to acquire the loc, k
g = Greenlet.spawn(self._acquire_lock, part)
self._part_lock_task_dict[part] = g
else:
# give up ownership of the partition
# cancel any lock acquisition which is ongoing
if (part in list(self._part_lock_task_dict.keys())):
try:
self._part_lock_task_dict[part].get(block=False)
except:
self._logger.error(
"canceling lock acquisition going on \
for:" + str(part))
# Cancelling the lock should result in killing the gevent
self._part_locks[part].cancel()
self._part_lock_task_dict[part].get(block=True)
del self._part_lock_task_dict[part]
if (part in self._curr_part_ownership_list):
release_lock_list.append(part)
self._curr_part_ownership_list.remove(part)
updated_curr_ownership = True
self._logger.error("giving up ownership of:" + str(part))
if (updated_curr_ownership is True):
# current partition membership was updated call the callback
self._update_cb(self._curr_part_ownership_list)
if (len(release_lock_list) != 0):
# release locks which were acquired
for part in release_lock_list:
self._logger.error("release the lock which was acquired:" + \
str(part))
try:
self._part_locks[part].release()
self._logger.error("fully gave up ownership of:" +
str(part))
except:
pass
from gevent import Greenlet
def thrFunc(s):
print "%s start.\n" % s
gevent.sleep(2)
print "%s end.\n" % s
gThr = Greenlet(thrFunc, "startjoin1")
gThr.start()
gThr.join()
gThr = Greenlet(thrFunc, "startjoin2")
gThr.start()
gThr.join()
print "+++++++++"
gThr = Greenlet(thrFunc, "start1")
gThr.start()
gThr = Greenlet(thrFunc, "start2")
gThr.start()
gThr = Greenlet(thrFunc, "start3")
gThr.start()
gThr = Greenlet(thrFunc, "start4")
gThr.start()
print "+++++++++"
gThr1 = Greenlet.spawn(thrFunc, "spawn1")
gThr2 = Greenlet.spawn(thrFunc, "spawn2")
gThr2.join()
def on_subscribe(self, msg):
"""
subscribe to a channel
The channels for this app are one of:
connected users
the current collection of interest
any model currently displayed
"""
def subscriber(io, topic):
"""
Subscribe to incoming pubsub messages from redis.
This will run in a greenlet, and blocks waiting for publish
messages from other redis clients. One source for the publish
events is a bridge to Django's signals - see colorpicks.publisher.
When a message is received on the redis channel, it emits an event
to backbone over socketio
"""
redis_sub = self.redis.pubsub()
redis_sub.subscribe(topic)
while io.socket.connected:
# TODO the nesting dictionaries with the key 'data'
# is highly problematic now - need to fix
for message in redis_sub.listen():
if message['type'] != 'message':
# We are only interested in 'message' events on the channel
# dlog( 'rejecting ', message)
continue
dlog(id(self), ' pubsub ', message)
# redis pubsub data is always a string
data = json.loads(message['data'])
if data['action'] == 'unsub':
redis_sub.unsubscribe(topic)
# ends the greenlet
return
colorid = str(data['data']['id'])
chan = message['channel']
if (data['action'] == 'delete'
and colorid == self.colorid):
# don't delete yourself, this can happen when you
# are watching the 'similar' channel, and you change
# your color
continue
if (data['action'] in ['update', 'create']
and self.show_only_connected_users
and not self.redis.sismember(
'connected_users', colorid)):
# a color of a non-connected user has been updated
# (perhaps through admin) and we are only watching for
# connected users - so do nothing
return
if chan == 'connected_users':
# we want to see if a connecting user is part of
# the current collection
if (self.redis.sismember(self.collection, colorid)
and self.show_only_connected_users):
# emit as if this were just added to the collection
chan = self.collection
else:
# don't emit a create event
continue
if not chan.startswith('color/'):
# again - hack to manage collection state on self here
chan = 'all'
dlog(id(self), 'emitting: ', chan, data['action'],
data['data'])
io.emit(chan + ":" + data['action'], data['data'])
if data['action'] == 'delete':
# send an extra delete event for the model also
# as the backbone collection seems buggy
io.emit('color/{}:delete'.format(data['data']['id']),
data['data'])
io.emit(chan + ":" + data['action'],
list(data['data']))
# we could filter our own ID out, so we don't subscribe to
# ourselves. It would depend on whether you want to allow changes
# made through other avenues to be reflected
# if you don't filter, that means there is no way to avoid
# getting your own round tripped updates - which defeates some of the
# point of the client side MVC
url = msg['url']
if url not in self.subscribers:
greenlet = Greenlet.spawn(subscriber, self, url)
# stash this greenlet in a dictionary in order
# to kill/unsub later on
self.subscribers[url] = greenlet
else:
pass
'xg',
'rb',
'hg',
'mlxy',
'tg',
'om',
'hx',
]
# 照片分类
photo_thread = [1, 2]
# 线程计数器
photo_number = -1
# 下载图片计数器，最大50
# index_number = 530273
# 页面计数器，最小530273，最大544527
file = '../photo/'
# 图片的保存地址
thread1 = Greenlet.spawn(photo_download, photo_thread[0], 530273,
photo_number, 0)
# 从命名中创建，并运行新的Greenlet的包装器
# 函数photo_download，带有传递的参数
thread2 = gevent.spawn(photo_download, photo_thread[1], 533836,
photo_number, 0)
# 两个thread运行，一个从530273页面开始爬取，另一个从537400页面开始爬取
# 537400 - 530273 = 7127
# 7127 / 2 = 3564
# 3564 + 530273 = 533836
threads = [thread1, thread2]
# 阻止所有线程完成
gevent.joinall(threads)
