def write(self, data): assert self._main is None self._buf.write(data) self._main = gevent.getcurrent() self._gen.switch() assert self._main == gevent.getcurrent() self._main = None
def interact(self):
_logger.info("Now engaged: [%s]", self.__c)
self.__send_hello()
self.__identify.enqueue(self)
# TODO(dustin): Something isn't robust enough. If we restart the server a
# couple of times, the client won't reconnect.
def terminate_cb(g):
# Technically, interact() will never return, so we'll never get here.
# But, this is logical.
self.__is_connected = False
if self.__ccallbacks is not None:
self.__ccallbacks.broken(self)
gevent.getcurrent().link(terminate_cb)
if self.__ccallbacks is not None:
self.__ccallbacks.connect(self)
gevent.spawn(self.__sender)
gevent.spawn(self.__receiver)
self.__send_thread_ev.wait()
_logger.debug("Sender has terminated. Waiting for receiver to "
"terminate.")
self.__receive_thread_ev.wait()
_logger.debug("Connection interaction has stopped (IGNORE_QUIT=[%s] "
"NICE_QUIT_EV=[%s] FORCE_QUIT_EV=[%s]): %s",
self.__ignore_quit, self.__nice_quit_ev.is_set(),
self.__force_quit_ev.is_set(), self)
def _acquire_data(cls, config, unlock_new_data_callback):
"""
Ensures required keys (such as stream_id) are available from config, configures the publisher and then calls:
BaseDataHandler._new_data_constraints (only if config does not contain 'constraints')
BaseDataHandler._publish_data passing BaseDataHandler._get_data as a parameter
@param config Dict containing configuration parameters, may include constraints, formatters, etc
@param unlock_new_data_callback BaseDataHandler callback function to allow conditional unlocking of the BaseDataHandler._semaphore
"""
stream_id = get_safe(config, 'stream_id')
if not stream_id:
raise ConfigurationError('Configuration does not contain required \'stream_id\' key')
#TODO: Configure the publisher
publisher=None
constraints = get_safe(config,'constraints')
if not constraints:
gevent.getcurrent().link(unlock_new_data_callback)
constraints = cls._new_data_constraints(config)
config['constraints']=constraints
cls._publish_data(publisher, config, cls._get_data(config))
# Publish a 'TestFinished' event
if get_safe(config,'TESTING'):
log.debug('Publish TestingFinished event')
pub = EventPublisher('DeviceCommonLifecycleEvent')
pub.publish_event(origin='BaseDataHandler._acquire_data', description='TestingFinished')
def _kill(self):
self.logging.info('Stopped Worker')
if self.threaded:
gevent.getcurrent().kill()
else:
sys.exit(1)
def wrapper(self, *args, **kwargs):
try:
return method(self, *args, **kwargs)
finally:
gevent.get_hub().handle_error = old
if self.peek_error()[0] is not None:
gevent.getcurrent().throw(*self.peek_error()[1:])
def _acquire_data(cls, config, publisher, unlock_new_data_callback):
"""
Ensures required keys (such as stream_id) are available from config, configures the publisher and then calls:
BaseDataHandler._new_data_constraints (only if config does not contain 'constraints')
BaseDataHandler._publish_data passing BaseDataHandler._get_data as a parameter
@param config Dict containing configuration parameters, may include constraints, formatters, etc
@param unlock_new_data_callback BaseDataHandler callback function to allow conditional unlocking of the BaseDataHandler._semaphore
"""
log.debug('start _acquire_data: config={0}'.format(config))
cls._init_acquisition_cycle(config)
constraints = get_safe(config,'constraints')
if not constraints:
gevent.getcurrent().link(unlock_new_data_callback)
constraints = cls._new_data_constraints(config)
if constraints is None:
raise InstrumentParameterException("Data constraints returned from _new_data_constraints cannot be None")
config['constraints'] = constraints
cls._publish_data(publisher, cls._get_data(config))
# Publish a 'TestFinished' event
if get_safe(config,'TESTING'):
log.debug('Publish TestingFinished event')
pub = EventPublisher('DeviceCommonLifecycleEvent')
pub.publish_event(origin='BaseDataHandler._acquire_data', description='TestingFinished')
def run(self):
logger.info('run: attempt %d - %s', self.attempt, gevent.getcurrent())
# Wrap main try block to catch failed attempt and call teardown before next attempt
try:
try:
if self.ready is not None:
logger.info('{} setup waiting'.format(self.__class__.__name__))
while not self.ready():
self.wait()
logger.info('{} setup ready'.format(self.__class__.__name__))
self.setup()
while True:
self.handle()
if _KILL_ALL_WORKERS:
break
except (KeyboardInterrupt, SystemExit, gevent.GreenletExit):
pass
except Exception as exc:
logger.error('Unrecoverable error %s: %r', gevent.getcurrent(), exc, exc_info=True)
sentry.captureException()
if self.attempt < self.max_attempts:
exc = WorkerAttemptFailed(self.__class__, self.attempt, original_exc=exc)
raise exc
finally:
logger.warning('tearing down greenlet %s', gevent.getcurrent())
with sentry.capture_on_error(raise_error=False):
self.teardown()
except WorkerAttemptFailed, exc:
self.wait()
exc.attempt_worker()
def wait(self, timeout=None):
if self.ready():
return self.value
else:
switch = getcurrent().switch
self.rawlink(switch)
try:
timer = Timeout.start_new(timeout) if timeout is not None else None
try:
getattr(getcurrent(), 'awaiting_batch', lambda: None)()
result = get_hub().switch()
assert result is self, 'Invalid switch into AsyncResult.wait(): %r' % (result, )
finally:
if timer is not None:
timer.cancel()
except Timeout as exc:
self.unlink(switch)
if exc is not timer:
raise
except:
self.unlink(switch)
raise
# not calling unlink() in non-exception case, because if switch()
# finished normally, link was already removed in _notify_links
return self.value
def call(self, method, message=None, timeout=None):
"""Send a message to the Actor this object addresses.
Wait for a result. If a timeout in seconds is passed, raise
gevent.TimeoutError if no result is returned in less than the timeout.
This could have nicer syntax somehow to make it look like an actual method call.
"""
message_id = str(uuid.uuid1())
my_address = gevent.getcurrent().address
self.cast(
{'call': message_id, 'method': method,
'address': my_address, 'message': message})
if timeout is None:
cancel = None
else:
## Raise any Timeout to the caller so they can handle it
cancel = gevent.Timeout(timeout)
cancel.start()
RSP = {'response': message_id, 'message': object}
EXC = {'response': message_id, 'exception': object}
INV = {'response': message_id, 'invalid_method': str}
pattern, response = gevent.getcurrent().receive(RSP, EXC, INV)
if cancel is not None:
cancel.cancel()
if pattern is INV:
raise RemoteAttributeError(method)
elif pattern is EXC:
raise RemoteException(response)
return response['message']
def _run_count_plugin(plugin):
"""Runs the count process for a plugin. Constantly polls for list of devices,
keeping an internal reference of the devices available from the plugin.
"""
count_identity = greenlet.random_ident()
e = event.add(count_identity, "BPPluginRegisterCount")
count_process_cmd = [plugin.executable_path, "--server_port=%s" %
config.get_value("server_address"), "--count"]
count_process = _start_process(count_process_cmd, count_identity)
if not count_process:
logging.warning("%s count process unable to start. removing.",
plugin.name)
return
try:
e.get(block=True, timeout=1)
except gevent.Timeout:
logging.info("%s count process never registered, removing.",
plugin.name)
return
except greenlet.BPGreenletExit:
logging.debug("Shutting down count process for %s", plugin.name)
return
logging.info("Count process for %s up on identity %s", plugin.name,
count_identity)
greenlet.add_identity_greenlet(count_identity, gevent.getcurrent())
hb = heartbeat.spawn_heartbeat(count_identity, gevent.getcurrent())
_plugins[plugin.name] = plugin
while True:
queue.add(count_identity, ["s", "BPPluginDeviceList"])
e = event.add(count_identity, "BPPluginDeviceList")
try:
(i, msg) = e.get(block=True, timeout=1)
except gevent.Timeout:
logging.info("%s count process timed out, removing.", plugin.name)
break
except greenlet.BPGreenletExit:
logging.debug("Shutting down count process for %s", plugin.name)
break
_devices[plugin.name] = msg[2]
# TODO: Make this a configuration value
try:
gevent.sleep(1)
except greenlet.BPGreenletExit:
logging.debug("Shutting down count process for %s", plugin.name)
break
# Heartbeat may already be dead if we're shutting down, so check first
if not hb.ready():
hb.kill(exception=greenlet.BPGreenletExit, block=True, timeout=1)
# Remove ourselves, but don't kill since we're already shutting down
greenlet.remove_identity_greenlet(count_identity, kill_greenlet=False)
# TODO: If a count process goes down, does every associated device go with
# it?
del _plugins[plugin.name]
queue.add(count_identity, ["s", "BPClose"])
logging.debug("Count process %s for %s exiting...", count_identity,
plugin.name)
def kill(self):
"""
This method can be used to kill the connection by killing the
Greenlet it is running in, use cautiously as this isn't a nice
way to disconnect clients and should only be used if the client
has already disconnected.
"""
gevent.getcurrent().kill(block=True)
def _operation_finish(self, source):
# WARNING: This method will be caused in it's own greenlet.
# Using self.monitor.execute from here will cause work to be owned by Root
# Cycle breaking
source.operation = None
# Setup the AsyncResult so *new* calls will return immediately
# But let's not notify the existing blocked greenlets until we
# have run the paradox detector
if source.successful():
self.result.value = source.value
self.result._exception = None
else:
self.result.value = None
self.result._exception = source.exception
# Purge any operations that were cached during a peek operation
checks = []
for p in self.peeks:
for c, op in p.walk_children():
if op.method.startswith("as_"):
checks.append(op)
op.purge_one()
if p.method.startswith("as_"):
checks.append(p)
p.purge_one()
getcurrent().operation = self
for op in checks:
try:
current_val = op.get()
new_val = self.monitor.wait(getattr(op.node, op.method))
except Exception as e:
self.result.set_exception(e)
return
if new_val != current_val:
self.result.set_exception(
errors.ParadoxError(
"Inconsistent configuration detected - changed from %r to %r" % (current_val, new_val),
anchor=op.node.anchor,
)
)
getcurrent().operation = None
return
getcurrent().operation = None
# Now notify all the other greenlets waiting for us that it is safe to continue
if source.successful():
self.result.set(source.value)
else:
self.result.set_exception(source.exception)
def f(n): for i in range(n): print gevent.getcurrent(), i g1 = gevent.spawn(f, 5) g2 = gevent.spawn(f, 5) g3 = gevent.spawn(f, 5) g1.join() g2.join() g3.join()
def _test_greenlet(self, link):
p = self.p
link(getcurrent())
assert len(p._links) == 1, p._links
p.unlink(getcurrent())
assert not p._links, p._links
g = gevent.Greenlet()
link(g)
assert len(p._links) == 1, p._links
p.unlink(g)
assert not p._links, p._links
def worker(pid):
print 'worker #%d-%s starting' % (pid,id(getcurrent()))
workerdata.cnt = 0
while True:
try:
task = taskqueue.get(timeout=5)
workerdata.cnt += 1
print 'worker #%d got task #%d' % (pid,task)
gevent.sleep(random.randint(0,3))
except Empty:
break
print 'worker #%d-%s exiting (cnt:%d)' % (pid,id(getcurrent()),workerdata.cnt)
def thread_lock(self, persist=False):
if self.sem.locked():
if self.locker == id(gevent.getcurrent()):
self.log.debug("Locker is -- %s. No need to re-lock", id(gevent.getcurrent()))
return False
else:
self.log.debug("Already locked by %s. Greenlet %s will wait...", self.locker, id(gevent.getcurrent()))
self.sem.acquire()
self.persist_lock = persist
self.locker = id(gevent.getcurrent())
self.log.debug("%s acquired lock (persist=%s)!", self.locker, persist)
return True
def acceptor(self, pool):
gevent.getcurrent()
while self.alive:
try:
conn, addr = self.socket.accept()
gt = pool.spawn(self.handle, conn, addr)
gt._conn = conn
gt.link(self.cleanup)
conn, addr, gt = None, None, None
except greenlet.GreenletExit:
return
except:
self.log.exception("Unexpected error in acceptor. Sepuku.")
os._exit(4)
def __worker(addr,log,workqueue,shutdown_event):
#def __worker(s,tag,src,dst,addr_type):
log.info("Worker {0} started".format(id(gevent.getcurrent())))
while True:
try:
callback,tag,src,dst,addr_type = workqueue.get(timeout=10)
except gevent.queue.Empty:
if shutdown_event.isSet():
log.info("Worker {0} shutting down".format(id(gevent.getcurrent())))
return
continue
data= {"type":"error","msg":"Incomplete"}
log.info("Worker got request for path {0}->{1}".format(src,dst))
s = gevent.socket.create_connection(addr)
if addr_type != "defined":
src = (src,addr_type)
dst = (dst,addr_type)
try:
req = ASQuerier._req
req.update(tag=tag,src=src,dst=dst)
s.sendall(json.dumps(req))
gevent.socket.wait_read(s.fileno(),timeout=180)
resp = s.recv(2048)
try:
data = json.loads(resp)
except ValueError:
data = {'type':'error',
'msg':"Failed to read response '{0}'".
format(resp)
}
if data['type'] != "error" and 'path' not in data:
data = {'type':'error',
'msg':"Response not understood '{0}'"
.format(resp)
}
except Exception as e:
sys.stderr.write("Caught exception {0}.\n".format(e))
sys.stderr.write("Returning data {1}\n".format(data))
finally:
s.close()
callback(data)
def nosleep_mgr():
old_switch_out = getattr(getcurrent(), 'switch_out', None)
def asserter():
raise ForbiddenYield("Context switch during `nosleep` region!")
getcurrent().switch_out = asserter
try:
yield
finally:
if old_switch_out is not None:
getcurrent().switch_out = old_switch_out
else:
del getcurrent().switch_out
def _acquire_sample(cls, config, publisher, unlock_new_data_callback, update_new_data_check_attachment):
"""
Ensures required keys (such as stream_id) are available from config, configures the publisher and then calls:
BaseDataHandler._constraints_for_new_request (only if config does not contain 'constraints')
BaseDataHandler._publish_data passing BaseDataHandler._get_data as a parameter
@param config Dict containing configuration parameters, may include constraints, formatters, etc
@param publisher the publisher used to publish data
@param unlock_new_data_callback BaseDataHandler callback function to allow conditional unlocking of the BaseDataHandler._semaphore
@param update_new_data_check_attachment classmethod to update the external dataset resources file list attachment
@throws InstrumentParameterException if the data constraints are not a dictionary
@retval None
"""
log.debug('start _acquire_sample: config={0}'.format(config))
cls._init_acquisition_cycle(config)
constraints = get_safe(config, 'constraints')
if not constraints:
gevent.getcurrent().link(unlock_new_data_callback)
try:
constraints = cls._constraints_for_new_request(config)
except NoNewDataWarning:
#log.info(nndw.message)
if get_safe(config, 'TESTING'):
#log.debug('Publish TestingFinished event')
pub = EventPublisher('DeviceCommonLifecycleEvent')
pub.publish_event(origin='BaseDataHandler._acquire_sample', description='TestingFinished')
return
if constraints is None:
raise InstrumentParameterException("Data constraints returned from _constraints_for_new_request cannot be None")
config['constraints'] = constraints
elif isinstance(constraints, dict):
addnl_constr = cls._constraints_for_historical_request(config)
if not addnl_constr is None and isinstance(addnl_constr, dict):
constraints.update(addnl_constr)
else:
raise InstrumentParameterException('Data constraints must be of type \'dict\': {0}'.format(constraints))
cls._publish_data(publisher, cls._get_data(config))
if 'set_new_data_check' in config:
update_new_data_check_attachment(config['external_dataset_res_id'], config['set_new_data_check'])
# Publish a 'TestFinished' event
if get_safe(config, 'TESTING'):
#log.debug('Publish TestingFinished event')
pub = EventPublisher('DeviceCommonLifecycleEvent')
pub.publish_event(origin='BaseDataHandler._acquire_sample', description='TestingFinished')
def return_socket(self):
if self.sock is None:
self.log.debug('No socket to return from greenlet %s', gevent.getcurrent() )
return
if self.sockets.qsize() < self.pool_size:
gl = gevent.getcurrent()
self.log.debug('Checkin socket %s from greenlet %s',
self.sock, gl)
self.sockets.put(self.sock)
self.sock = None
else:
self.log.debug('Close socket in greenlet %s', gevent.getcurrent() )
self.sock.close()
self.sock = None
self.local.sock = None
def _kill(self):
# Garbage collect the sockets to avoid weirdness
self.control_socket.close()
self.pull_socket.close()
self.push_socket.close()
self.control_socket = None
self.pull_socket = None
self.push_socket = None
self.logging.info('Stopped Worker')
if self.threaded:
gevent.getcurrent().kill()
else:
sys.exit(1)
def step_actor(self, actor):
# Pretend that the current greenlet is the Actor to bypass
# actor_message's asserts.
with mock.patch.object(actor, "greenlet"):
actor.greenlet = gevent.getcurrent()
while not actor._event_queue.empty():
actor._step()
def run(self, path, file):
try:
self.first = self.current
with transaction:
file.greenlet = gevent.getcurrent()
file.on_greenlet_started()
try:
result = self.bruteforce(path, file)
except rarfile.NeedFirstVolume:
self.next = os.path.join(path.dir, "{}.part{}.rar".format(path.basename, "1".zfill(len(path.part))))
return self.find_next()
if result and result is not True:
raise result
if self.password:
rarpw = "-p"+self.password
else:
rarpw = "-p-"
cmd = [rarfile.UNRAR_TOOL, "x", "-y", rarpw, "-idq", "-vp", path, file.get_extract_path() + os.sep]
file.log.info("starting extraction of {} with params {}".format(path[1:], cmd))
self.rar = rarfile.custom_popen(cmd)
self.wait_data()
if not path.finished.ready():
path.finished.set()
except BaseException as e:
traceback.print_exc()
self.kill(e)
raise
def _send_loop(self):
self._send_greenlet = gevent.getcurrent()
msg = None
c = 0
try:
while True:
if c >= self.COUNT:
c = 0
self._stop_send()
msg = self._send_queue.get()
self._check_send_connection()
try:
self._push_connection.send(str(msg))
#logging.debug('[apns]send:%s', msg)
c += 1
gevent.sleep(0.05)
#print '_send_loop:', msg
except (Exception, error) as e:
logging.error('[apns]send error:%s, token:%s, id:%s, left:%s',
e, msg.token, msg.identifier, self._send_queue.qsize())
self._send_queue.put(msg)
self._stop_send()
finally:
if self._send_queue.qsize() < 1:
logging.info('[apns]send finish')
if not self._send_queue_cleared.is_set():
self._send_queue_cleared.set()
except gevent.GreenletExit, e:
pass
def connection_cache_key(params):
'''params to connection cache key tuple: (None, host, user, key)'''
try:
return id(getcurrent()), params['hostname'], params['ssh']['user'], params['ssh']['keyfile']
except KeyError as err:
# some key is missing in params
raise ConnectionCacheError('params missing key: %s' % err)
def stop(self, timeout=None):
"""Stop this service and child services
If the server uses a pool to spawn the requests, then :meth:`stop` also waits
for all the handlers to exit. If there are still handlers executing after *timeout*
has expired (default 1 second), then the currently running handlers in the pool are killed."""
if gevent.getcurrent() in self._greenlets:
return gevent.spawn(self.stop)
self.started = False
try:
self.pre_stop()
for child in reversed(self._children):
# iterate over children in reverse order, in case dependancies
# were implied by the starting order
if child.started:
child.stop()
self.do_stop()
finally:
if timeout is None:
timeout = self.stop_timeout
if self._greenlets:
self._greenlets.join(timeout=timeout)
self._greenlets.kill(block=True, timeout=1)
self._ready_event.clear()
self._stopped_event.set()
self.post_stop()
def test_multiple_listeners_error(self):
# if there was an error while calling a callback
# it should not prevent the other listeners from being called
# also, all of the errors should be logged, check the output
# manually that they are
p = gevent.spawn(lambda: 5)
results = []
def listener1(*args):
results.append(10)
raise ExpectedError('listener1')
def listener2(*args):
results.append(20)
raise ExpectedError('listener2')
def listener3(*args):
raise ExpectedError('listener3')
p.link(listener1)
p.link(listener2)
p.link(listener3)
sleep(DELAY * 10)
assert results in [[10, 20], [20, 10]], results
p = gevent.spawn(lambda: getcurrent().throw(ExpectedError('test_multiple_listeners_error')))
results = []
p.link(listener1)
p.link(listener2)
p.link(listener3)
sleep(DELAY * 10)
assert results in [[10, 20], [20, 10]], results
def get(self):
try:
return [value.get() for value in self.seq]
except Exception as e:
import gevent
print "xxx", gevent.getcurrent()
raise e
def _invalidate_connections(self, key=None):
if key is None:
key = getcurrent()
connections = self._get_key_connections(key)
uniq = set([x[0] for x in connections.values()])
for conn in uniq:
conn.invalidate()
def f2(n):
for i in range(n):
print gevent.getcurrent(), i
gevent.sleep(0)
def _get_current_context(self):
"""Helper to get the current context from the current greenlet"""
current_g = gevent.getcurrent()
if current_g is not None:
return getattr(current_g, CONTEXT_ATTR, None)
return None
def clear_context():
gevent.getcurrent().neutron_api_context = None
monkey.patch_all()
from gevent.server import StreamServer
import logging
import sys
'''
# --- for dbg
from gevent import signal as sig
import signal
sig(signal.SIGUSR1, lambda: False)
# -----------
'''
MAIN = gevent.getcurrent()
from gevent import signal as sig
import signal
def _exit_handler(*a, **k):
gevent.kill(MAIN, SystemExit)
sig(signal.SIGTERM, _exit_handler)
from game import autoenv
import argparse
parser = argparse.ArgumentParser(prog=sys.argv[0])
parser.add_argument('node', type=str)
parser.add_argument('--port', default=9999, type=int)
def _get_greenlet_scope(self, greenlet=None):
if greenlet is None:
greenlet = gevent.getcurrent()
return getattr(greenlet, ACTIVE_ATTR, None)
def test_raise(self):
p = gevent.spawn(lambda: getcurrent().throw(ExpectedError('test_raise')))
for _ in range(3):
self._test_raise(p)
def func():
child = greenlet.Greenlet()
self.assertIs(child.spawn_tree_locals, getcurrent().spawn_tree_locals)
self.assertIs(child.spawning_greenlet(), getcurrent())
def invoke(self, func, args, kwargs):
# put the process in the host list pthreads
self.host.pthreads[getcurrent()] = self.__actor.url
func(*args, **kwargs)
# remove the process from pthreads
del self.host.pthreads[getcurrent()]
def task(n):
for i in range(n):
print("执行 %s => %d" % (gevent.getcurrent(), i))
time.sleep(0.2)
def f1(n):
for i in range(n):
print(gevent.getcurrent(), i)
# time.sleep(0.5)
gevent.sleep(0.5)
def zap_loop(self, sender, **kwargs):
self._is_connected = True
self._zap_greenlet = gevent.getcurrent()
sock = self.zap_socket
time = gevent.core.time
blocked = {}
wait_list = []
timeout = None
if self.core.messagebus == 'rmq':
# Check the topic permissions of all the connected agents
self._check_rmq_topic_permissions()
else:
self._send_protected_update_to_pubsub(self._protected_topics)
while True:
events = sock.poll(timeout)
now = time()
if events:
zap = sock.recv_multipart()
version = zap[2]
if version != b'1.0':
continue
domain, address, userid, kind = zap[4:8]
credentials = zap[8:]
if kind == b'CURVE':
credentials[0] = encode_key(credentials[0])
elif kind not in [b'NULL', b'PLAIN']:
continue
response = zap[:4]
user = self.authenticate(domain, address, kind, credentials)
_log.debug("AUTH: authenticated user id: {0}, {1}".format(
user, userid))
if user:
_log.info(
'authentication success: domain=%r, address=%r, '
'mechanism=%r, credentials=%r, user_id=%r', domain,
address, kind, credentials[:1], user)
response.extend([b'200', b'SUCCESS', user, b''])
sock.send_multipart(response)
else:
_log.info(
'authentication failure: domain=%r, address=%r, '
'mechanism=%r, credentials=%r', domain, address, kind,
credentials)
# If in setup mode, add/update auth entry
if self._setup_mode:
self._update_auth_entry(domain, address, kind,
credentials[0], userid)
_log.info(
'new authentication entry added in setup mode: domain=%r, address=%r, '
'mechanism=%r, credentials=%r, user_id=%r', domain,
address, kind, credentials[:1], userid)
response.extend([b'200', b'SUCCESS', '', b''])
_log.debug("AUTH response: {}".format(response))
sock.send_multipart(response)
else:
self._update_auth_failures(domain, address, kind,
credentials[0], userid)
try:
expire, delay = blocked[address]
except KeyError:
delay = random.random()
else:
if now >= expire:
delay = random.random()
else:
delay *= 2
if delay > 100:
delay = 100
expire = now + delay
bisect.bisect(wait_list, (expire, address, response))
blocked[address] = expire, delay
while wait_list:
expire, address, response = wait_list[0]
if now < expire:
break
wait_list.pop(0)
response.extend([b'400', b'FAIL', b'', b''])
sock.send_multipart(response)
try:
if now >= blocked[address][0]:
blocked.pop(address)
except KeyError:
pass
timeout = (wait_list[0][0] - now) if wait_list else None
def _after_call(self):
if not self.call_link_rpc:
return
gl = getcurrent()
gl._game_client_rpc_ = None
def gclear(self):
pass
data = open(options.replay_file, 'r').read().split('\n')
while data[0].startswith('#'):
print data.pop(0)
mode = data.pop(0)
rndseed = long(data.pop(0))
gdlist = json.loads(data.pop(0))
datasource = DataSource(gdlist)
datasource.start()
from gamepack import gamemodes
mode = gamemodes[mode]
clients = [MockClient(i, datasource) for i in xrange(mode.n_persons)]
players = PlayerList([Player(i) for i in clients])
g = mode()
g.players = players
g.rndseed = rndseed
g.random = random.Random(rndseed)
gevent.getcurrent().game = g
g._run()
def get_cur_rpc():
gl = getcurrent()
return getattr(gl, '_game_client_rpc_', None)
def _set_greenlet_scope(self, scope, greenlet=None):
if greenlet is None:
greenlet = gevent.getcurrent()
setattr(greenlet, ACTIVE_ATTR, scope)
def f(n):
for i in range(n):
print(gevent.getcurrent(), i)
def _before_call(self):
if not self.call_link_rpc:
return
gl = getcurrent()
gl._game_client_rpc_ = self
def f2(n):
for i in range(n):
print(gevent.getcurrent(), i)
gevent.sleep(0.5)
def have_context():
return (hasattr(gevent.getcurrent(), 'neutron_api_context')
and gevent.getcurrent().neutron_api_context is not None)
def set_context(api_ctx):
gevent.getcurrent().neutron_api_context = api_ctx
def get_current_job(greenlet_id=None):
if greenlet_id is None:
greenlet_id = id(gevent.getcurrent())
return _GLOBAL_CONTEXT["greenlets"].get(greenlet_id)
def handler():
gevent.getcurrent().name = name
logger.info(f"Received {signal.Signals(signum).name} signal.")
f(*args, **kwargs)
def _save_status(self, status, updates=None, exception=False, w=None, j=None):
if self.id is None:
return
context.metric("jobs.status.%s" % status)
if self.stored is False and self.statuses_no_storage is not None and status in self.statuses_no_storage:
return
now = datetime.datetime.utcnow()
db_updates = {
"status": status,
"dateupdated": now
}
db_updates.update(updates or {})
if self.datestarted:
db_updates["totaltime"] = (now - self.datestarted).total_seconds()
if context.get_current_config().get("trace_greenlets"):
current_greenlet = gevent.getcurrent()
# TODO are we sure the current job is doing the save_status() on itself?
if hasattr(current_greenlet, "_trace_time"):
# pylint: disable=protected-access
db_updates["time"] = current_greenlet._trace_time
db_updates["switches"] = current_greenlet._trace_switches
if exception:
trace = traceback.format_exc()
context.log.error(trace)
db_updates["traceback"] = trace
exc = sys.exc_info()[0]
db_updates["exceptiontype"] = exc.__name__
if context.get_current_config().get("save_traceback_history"):
self._save_traceback_history(status, trace, exc)
# In the most common case, we allow an optimization on Mongo writes
if status == "success":
if w is None:
w = getattr(self.task, "status_success_update_w", None)
if j is None:
j = getattr(self.task, "status_success_update_j", None)
# This job wasn't inserted because "started" is in statuses_no_storage
# So we must insert it for the first time instead of updating it.
if self.stored is False:
db_updates["queue"] = self.data["queue"]
db_updates["params"] = self.data["params"]
db_updates["path"] = self.data["path"]
self.collection.insert(db_updates, w=w, j=j, manipulate=True)
self.id = db_updates["_id"] # Persistent ID assigned by the server
self.stored = True
else:
self.collection.update({
"_id": self.id
}, {"$set": db_updates}, w=w, j=j, manipulate=False)
if self.data:
self.data.update(db_updates)
def perform(self):
""" Loads and starts the main task for this job, the saves the result. """
if self.data is None:
return
context.log.debug("Starting %s(%s)" % (self.data["path"], self.data["params"]))
task_class = load_class_by_path(self.data["path"])
self.task = task_class()
self.task.is_main_task = True
try:
lock = None
if self.task.max_concurrency:
if self.task.max_concurrency > 1:
raise NotImplementedError()
# TODO: implement a semaphore
lock = context.connections.redis.lock(self.redis_max_concurrency_key, timeout=self.timeout + 5)
if not lock.acquire(blocking=True, blocking_timeout=0):
raise MaxConcurrencyInterrupt()
result = self.task.run_wrapped(self.data["params"])
finally:
if lock:
try:
lock.release()
except LockError:
pass
self.save_success(result)
if context.get_current_config().get("trace_greenlets"):
# TODO: this is not the exact greenlet_time measurement because it doesn't
# take into account the last switch's time. This is why we force a last switch.
# This does cause a performance overhead. Instead, we should print the
# last timing directly from the trace() function in context?
# pylint: disable=protected-access
gevent.sleep(0)
current_greenlet = gevent.getcurrent()
t = (datetime.datetime.utcnow() - self.datestarted).total_seconds()
context.log.debug(
"Job %s success: %0.6fs total, %0.6fs in greenlet, %s switches" %
(self.id,
t,
current_greenlet._trace_time,
current_greenlet._trace_switches - 1)
)
else:
context.log.debug("Job %s success: %0.6fs total" % (
self.id, (datetime.datetime.utcnow() -
self.datestarted).total_seconds()
))
return result
def get_context():
return gevent.getcurrent().neutron_api_context
def zap_loop(self, sender, **kwargs):
self._is_connected = True
self._zap_greenlet = gevent.getcurrent()
sock = self.zap_socket
time = gevent.core.time
blocked = {}
wait_list = []
timeout = None
while True:
events = sock.poll(timeout)
now = time()
if events:
zap = sock.recv_multipart()
version = zap[2]
if version != b'1.0':
continue
domain, address, _, kind = zap[4:8]
credentials = zap[8:]
if kind == b'CURVE':
credentials[0] = encode_key(credentials[0])
elif kind not in [b'NULL', b'PLAIN']:
continue
response = zap[:4]
user = self.authenticate(domain, address, kind, credentials)
if user:
_log.info(
'authentication success: domain=%r, address=%r, '
'mechanism=%r, credentials=%r, user_id=%r', domain,
address, kind, credentials[:1], user)
response.extend([b'200', b'SUCCESS', user, b''])
sock.send_multipart(response)
else:
_log.info(
'authentication failure: domain=%r, address=%r, '
'mechanism=%r, credentials=%r', domain, address, kind,
credentials)
try:
expire, delay = blocked[address]
except KeyError:
delay = random.random()
else:
if now >= expire:
delay = random.random()
else:
delay *= 2
if delay > 100:
delay = 100
expire = now + delay
bisect.bisect(wait_list, (expire, address, response))
blocked[address] = expire, delay
while wait_list:
expire, address, response = wait_list[0]
if now < expire:
break
wait_list.pop(0)
response.extend([b'400', b'FAIL', b'', b''])
sock.send_multipart(response)
try:
if now >= blocked[address][0]:
blocked.pop(address)
except KeyError:
pass
timeout = (wait_list[0][0] - now) if wait_list else None
def f(n):
for i in range(n):
print(gevent.getcurrent(), i)
#用来模拟一个耗时操作
gevent.sleep(1.0)
def fun(num):
for i in range(num):
print('%s%s'%(gevent.getcurrent(),str(i)))
gevent.sleep(1) #利用gevent中的sleep方法模拟耗时操作
def f(n):
for i in range(n):
print( gevent.getcurrent(), i)
# ⽤来模拟⼀个耗时操作，注意不是time模块中的sleep
gevent.sleep(1)
def coroutine_work(work_no, cycle_index):
"""协程任务"""
for i in range(cycle_index):
print("%s模拟IO操作：%d" % (work_no, i), gevent.getcurrent())
time.sleep(1)
