def start(num_of_clients):
task_pool = []
for i in range(num_of_clients):
cookie_str = datetime.datetime.now().strftime("%H%M%S%f")
task = gevent.spawn(send_heartbeat, i, SHOW_ID, cookie_str, CLIENT_SLEEP_TIME)
task_pool.append(task)
gevent.wait(task_pool)
def exec_in_coroutine(self, stream, ident, parent):
import gevent
task = gevent.spawn(_execute_request, self, stream, ident, parent)
while not task.value:
gevent.wait(timeout=0.01)
self.frontends[0].iopub_channel.process_events()
def stop_and_wait(self):
# Stop handling incoming packets, but don't close the socket. The
# socket can only be safely closed after all outgoing tasks are stopped
self.server.stop_accepting()
# Stop processing the outgoing queues
self.event_stop.set()
gevent.wait(self.greenlets)
# All outgoing tasks are stopped. Now it's safe to close the socket. At
# this point there might be some incoming message being processed,
# keeping the socket open is not useful for these.
self.server.stop()
# Calling `.close()` on a gevent socket doesn't actually close the underlying os socket
# so we do that ourselves here.
# See: https://github.com/gevent/gevent/blob/master/src/gevent/_socket2.py#L208
# and: https://groups.google.com/forum/#!msg/gevent/Ro8lRra3nH0/ZENgEXrr6M0J
try:
self.server._socket.close() # pylint: disable=protected-access
except socket.error:
pass
# Set all the pending results to False
for async_result in self.messageids_to_asyncresults.values():
async_result.set(False)
def execute_from_cli(hostname, cluster, with_ui, peers):
polar_client = PolarClient(hosts=peers, handler=GeventHandler())
polar_client.start()
crow = CrowContext(polar_client, hostname, cluster)
actor_name = "%s-%s" % (hostname, cluster)
actor_system = ActorSystem(
polar_client=polar_client,
name=actor_name
)
actor_system.actor_of(KeepRpcActor.props(
crow=crow,
polar_client=polar_client
))
crow.announce()
if with_ui == True:
from keep.ui import app
app.config["actor_flask"] = actor_system
app.config["polar_client"] = polar_client
# adapt polar client wo work with the WSGI adapter
http_server = WSGIServer(('', 5000), app)
print "Stared UI on port 5000"
http_server.serve_forever()
gevent.wait()
def start_listeners(self):
"""
Starts all listeners in managed greenlets.
Usually called by the ProcManager, unless using IonProcess manually.
"""
try:
# disable normal error reporting, this method should only be called from startup
self.thread_manager._failure_notify_callback = None
# spawn all listeners in startup listeners (from initializer, or added later)
for listener in self._startup_listeners:
self.add_endpoint(listener)
with Timeout(seconds=CFG.get_safe('container.messaging.timeout.start_listener', 30)):
gevent.wait([x.get_ready_event() for x in self.listeners])
except Timeout:
# remove failed endpoints before reporting failure above
for listener, proc in self._listener_map.iteritems():
if proc.proc.dead:
log.info("removed dead listener: %s", listener)
self.listeners.remove(listener)
self.thread_manager.children.remove(proc)
raise IonProcessError("start_listeners did not complete in expected time")
finally:
self.thread_manager._failure_notify_callback = self._child_failed
def test_rawlink_on_unacquired_runs_notifiers(self):
# https://github.com/gevent/gevent/issues/1287
# Rawlinking a ready semaphore should fire immediately,
# not raise LoopExit
s = Semaphore()
gevent.wait([s])
def run(self, actorlist):
for actor in actorlist:
assert command.has_cmd(actor["cmd"]),actor["cmd"]
for _ in xrange(actor.get("count", 1)):
self.workers.append(gevent.spawn(command.do_cmd, Game(self.srv_addr), actor["cmd"], actor["args"]))
gevent.wait(self.workers)
def get_biobrick_data():
def get_data(obj):
name = obj.to_dict()['name']
print 'Trying to get information of %s ...' % name
fp = urlopen('http://parts.igem.org/cgi/xml/part.cgi?part=' + name)
doc = ET.parse(fp)
part = doc.find('.//part')
if part is None:
print 'Not found: %s.' % name
return
print 'Success: %s.' % name
part_id = part.find('./part_id').text
short_name = part.find('./part_short_name').text
nickname = part.find('./part_nickname').text or ''
description = part.find('./part_short_desc').text
sequence = part.find('./sequences/seq_data').text.replace('\n', '')
obj.part_id = int(part_id)
obj.short_name = short_name
obj.nickname = nickname
obj.description = description
obj.sequence = sequence
jobs = []
jobs.extend(gevent.spawn(get_data, obj) for obj in Receptor.query)
jobs.extend(gevent.spawn(get_data, obj) for obj in Promoter.query)
jobs.extend(gevent.spawn(get_data, obj) for obj in RBS.query)
jobs.extend(gevent.spawn(get_data, obj) for obj in Terminator.query)
jobs.extend(gevent.spawn(get_data, obj) for obj in Output.query)
gevent.wait(jobs)
db.session.commit()
def test_direct_transfer_to_offline_node(raiden_network, token_addresses):
token_address = token_addresses[0]
app0, app1 = raiden_network
# Wait until the initialization of the node is complete and then stop it
gevent.wait([app1.raiden.start_event])
app1.raiden.stop()
amount = 10
target = app1.raiden.address
async_result = app0.raiden.direct_transfer_async(
token_address,
amount,
target,
identifier=1,
)
assert async_result.wait(5) is None
app1.raiden.start()
assert async_result.wait(5) is True
assert_mirror(
channel(app0, app1, token_address),
channel(app1, app0, token_address),
)
def query(self):
jobs = [gevent.spawn(self.do_query, i) for i in xrange(self.num_)]
gevent.wait(jobs)
self.out_put_.extend([self.process_index_, reduce(
lambda x, y: x + y, self.before_merge_res_)])
pass
def mainloop(sup):
try:
gevent.wait()
except KeyboardInterrupt:
pass
finally:
sup.stop()
def resources():
group = gevent.pool.Group()
source_view = gevent.spawn(
flask.copy_current_request_context(cloud_view),
hooks.source,
)
if "destination" not in flask.current_app.config["CLOUDS"]:
gevent.wait([source_view])
return flask.jsonify(
reset=flask.current_app.config["CLOUDS_RESET"],
source=source_view.value,
# TODO(akscram): A set of hosts that don't belong to any cloud.
hosts=[],
# TODO(akscram): A set of current events.
events=[],
)
destination_view = gevent.spawn(
flask.copy_current_request_context(cloud_view),
hooks.destination,
)
gevent.wait([source_view, destination_view])
return flask.jsonify(
reset=flask.current_app.config["CLOUDS_RESET"],
source=source_view.value,
destination=destination_view.value,
# TODO(akscram): A set of hosts that don't belong to any cloud.
hosts=[],
# TODO(akscram): A set of current events.
events=[],
)
def main():
parser = argparse.ArgumentParser('emfas')
parser.add_argument('--api-key', required=True, help='moomash api key')
parser.add_argument('-v', '--verbose', action='count')
parser.add_argument('url', help='twitch url')
ns = parser.parse_args()
emfas = Emfas(ns.api_key)
sp = TwitchSegmentProvider2(ns.url)
emfas.start(sp)
if ns.verbose > 0:
setup_logging()
if ns.verbose < 1:
logging.getLogger('requests').setLevel(logging.WARNING)
if ns.verbose < 2:
sp.ls.set_loglevel(logging.WARNING)
if ns.verbose < 3:
logging.getLogger('requests.packages.urllib3.connectionpool')\
.setLevel(logging.WARNING)
def every_minute():
songs = emfas.identify()
print 'Songs:', ', '.join(map(unicode, songs))
gevent.spawn_later(60, every_minute)
gevent.spawn_later(60, every_minute)
gevent.wait([emfas._worker])
def stop(self):
""" Stop the node. """
# Needs to come before any greenlets joining
self.stop_event.set()
self.transport.stop_and_wait()
self.alarm.stop_async()
wait_for = [self.alarm]
wait_for.extend(getattr(self.transport, 'greenlets', []))
# We need a timeout to prevent an endless loop from trying to
# contact the disconnected client
gevent.wait(wait_for, timeout=self.shutdown_timeout)
# Filters must be uninstalled after the alarm task has stopped. Since
# the events are polled by an alarm task callback, if the filters are
# uninstalled before the alarm task is fully stopped the callback
# `poll_blockchain_events` will fail.
#
# We need a timeout to prevent an endless loop from trying to
# contact the disconnected client
try:
with gevent.Timeout(self.shutdown_timeout):
self.blockchain_events.uninstall_all_event_listeners()
except (gevent.timeout.Timeout, RaidenShuttingDown):
pass
self.blockchain_events.reset()
if self.db_lock is not None:
self.db_lock.release()
def stress_send_and_receive_parallel_transfers(
rest_apis,
token_address,
identifier_generator,
deposit,
):
"""Send transfers of value one in parallel"""
pairs = list(zip(rest_apis, rest_apis[1:] + [rest_apis[0]]))
foward_transfers = [
gevent.spawn(
sequential_transfers,
server_from=server_from,
server_to=server_to,
number_of_transfers=deposit,
token_address=token_address,
identifier_generator=identifier_generator,
)
for server_from, server_to in pairs
]
backwards_transfers = [
gevent.spawn(
sequential_transfers,
server_from=server_from,
server_to=server_to,
number_of_transfers=deposit,
token_address=token_address,
identifier_generator=identifier_generator,
)
for server_to, server_from in pairs
]
gevent.wait(foward_transfers + backwards_transfers)
def stop(self):
""" Stop the node. """
# Needs to come before any greenlets joining
self.stop_event.set()
self.protocol.stop_and_wait()
self.alarm.stop_async()
wait_for = [self.alarm]
wait_for.extend(self.protocol.greenlets)
wait_for.extend(self.greenlet_task_dispatcher.stop())
# We need a timeout to prevent an endless loop from trying to
# contact the disconnected client
gevent.wait(wait_for, timeout=self.shutdown_timeout)
# Filters must be uninstalled after the alarm task has stopped. Since
# the events are polled by an alarm task callback, if the filters are
# uninstalled before the alarm task is fully stopped the callback
# `poll_blockchain_events` will fail.
#
# We need a timeout to prevent an endless loop from trying to
# contact the disconnected client
try:
with gevent.Timeout(self.shutdown_timeout):
self.blockchain_events.uninstall_all_event_listeners()
except (gevent.timeout.Timeout, RaidenShuttingDown):
pass
# save the state after all tasks are done
if self.serialization_file:
save_snapshot(self.serialization_file, self)
if self.db_lock is not None:
self.db_lock.release()
def restart_network(raiden_network, retry_timeout):
for app in raiden_network:
app.stop()
wait_network = [
gevent.spawn(restart_app, app)
for app in raiden_network
]
gevent.wait(wait_network)
new_network = [
greenlet.get()
for greenlet in wait_network
]
# The tests assume the nodes are available to transfer
for app0, app1 in combinations(new_network, 2):
waiting.wait_for_healthy(
app0.raiden,
app1.raiden.address,
retry_timeout,
)
return new_network
def run(self):
ob = ObserverManager(11011)
ob.start()
p = PlayerManager(11012)
p.start()
gevent.wait()
def test_regression_unfiltered_routes(
raiden_network,
token_addresses,
settle_timeout,
deposit,
):
""" The transfer should proceed without triggering an assert.
Transfers failed in networks where two or more paths to the destination are
possible but they share same node as a first hop.
"""
app0, app1, app2, app3, app4 = raiden_network
token = token_addresses[0]
registry_address = app0.raiden.default_registry.address
# Topology:
#
# 0 -> 1 -> 2 -> 4
# | ^
# +--> 3 ---+
app_channels = [
(app0, app1),
(app1, app2),
(app1, app3),
(app3, app4),
(app2, app4),
]
greenlets = []
for first_app, second_app in app_channels:
greenlets.append(gevent.spawn(
payment_channel_open_and_deposit,
first_app,
second_app,
token,
deposit,
settle_timeout,
))
gevent.wait(greenlets)
wait_for_channels(
app_channels,
registry_address,
[token],
deposit,
)
payment_network_identifier = app0.raiden.default_registry.address
token_network_identifier = views.get_token_network_identifier_by_token_address(
views.state_from_app(app0),
payment_network_identifier,
token,
)
transfer = app0.raiden.mediated_transfer_async(
token_network_identifier=token_network_identifier,
amount=1,
target=app4.raiden.address,
identifier=1,
)
assert transfer.wait()
def run(self):
self.server = SpamdTCP(self.address)
# self.port = server.server_port
gevent.signal(signal.SIGTERM, self.server.close)
gevent.signal(signal.SIGINT, self.server.close)
self.server.start()
gevent.wait()
def run(self, stdin, log):
jobs = [gevent.spawn(service.load()) for service
in pkg.get_entry_points("canopy.kaleidoscope")]
try:
gevent.wait(jobs)
except KeyboardInterrupt:
print("killed abruptly")
def main():
"""
Import plugin modules. Find subclasses of Plugin. Use those subclasses.
"""
modules = import_plugins()
plugins = find_plugins()
jobs = launch_plugins(plugins)
gevent.wait(jobs)
def server_forever(listener):
global service
log.info('worker [%d] start',os.getpid())
service = GStreamServer( listener, processor = processor, inputTransportFactory = tfactory, inputProtocolFactory = pfactory, spawn=max_conn)
signal.signal(signal.SIGTERM, signal_worker_handler)
service.start()
gevent.wait()
log.info('worker [%d] exit', os.getpid())
def targets(self, activity):
activities = self.get_contacts_by_activity[activity['id']]
contacts = [int(c) for c in activities[TARGETS]]
pool = ThreadPool(THREADS)
contacts = [pool.spawn(self.get_contact, c) for c in contacts]
gevent.wait()
contacts = [c.get()['sort_name'] for c in contacts]
return ', '.join(contacts)
def __call__(self):
self.change_permissions()
periodical_tasks = []
if self.args['--recalculate']:
def after_task(periodical_task):
periodical_task.task.fill_farm_usage_d(force=True)
periodical_task.stop()
if 'poller' in self.config['billing']:
poller_task = billing.RecalculatePollerBilling(self.analytics, self.config)
periodical_tasks.append(helper.PeriodicalTask(poller_task,
timeout=60 * 60 * 6,
after=after_task))
if 'aws-detailed-billing' in self.config['billing']:
aws_billing_task = billing.RecalculateAWSBilling(self.analytics, self.config)
periodical_tasks.append(helper.PeriodicalTask(aws_billing_task,
timeout=60 * 60 * 12,
after=after_task))
else:
def after_task(periodical_task):
periodical_task.task.fill_farm_usage_d()
if self.config['dtime_to']:
periodical_task.stop()
else:
periodical_task.task.config['dtime_from'] = False
if 'poller' in self.config['billing']:
poller_task = billing.PollerBilling(self.analytics, self.config)
periodical_tasks.append(helper.PeriodicalTask(poller_task,
period=poller_period,
timeout=poller_timeout,
after=after_task))
if 'aws-detailed-billing' in self.config['billing']:
aws_task = billing.AWSBilling(self.analytics, self.config)
periodical_tasks.append(helper.PeriodicalTask(aws_task,
period=aws_period,
timeout=aws_timeout,
after=after_task))
if 'azure' in self.config['billing']:
azure_task = billing.AzureBilling(self.analytics, self.config)
periodical_tasks.append(helper.PeriodicalTask(azure_task,
period=azure_period,
timeout=azure_timeout,
after=after_task))
results = []
for periodical_task in periodical_tasks:
results.append(periodical_task())
if len(periodical_tasks) > 1:
time.sleep(launch_delay)
gevent.wait(results)
def _work(self, burst=False):
"""Starts the work loop.
Pops and performs all jobs on the current list of queues. When all
queues are empty, block and wait for new jobs to arrive on any of the
queues, unless `burst` mode is enabled.
The return value indicates whether any jobs were processed.
"""
setup_loghandlers()
self._install_signal_handlers()
self.did_perform_work = False
self.register_birth()
self.log.info(
"RQ gevent worker (greenlet pool size={0}) {1!r} started, version {2}".format(
self.gevent_pool.size, self.key, VERSION
)
)
self.set_state(WorkerStatus.STARTED)
try:
while True:
try:
self.check_for_suspension(burst)
if self.should_run_maintenance_tasks:
self.clean_registries()
if self._stop_requested:
self.log.info("Stopping on request.")
break
timeout = None if burst else max(1, self.default_worker_ttl - 60)
result = self.dequeue_job_and_maintain_ttl(timeout)
if result is None and burst:
self.log.info("RQ worker {0!r} done, quitting".format(self.key))
try:
# Make sure dependented jobs are enqueued.
gevent.wait(self.gevent_greenlets)
except LoopExit:
pass
result = self.dequeue_job_and_maintain_ttl(timeout)
if result is None:
break
except StopRequested:
break
job, queue = result
self.execute_job(job, queue)
finally:
if not self.is_horse:
self.register_death()
return self.did_perform_work
def wait(self, tasks, timeout=None, count=None):
"""
Wait for tasks to finish with optional timeout.
"""
assert isinstance(tasks, list)
assert len(tasks) > 0
gevent.wait(tasks, timeout, count)
def run_many(tests, expected=None, failfast=False):
global NWORKERS, pool
start = time()
total = 0
failed = {}
NWORKERS = min(len(tests), NWORKERS)
pool = Pool(NWORKERS)
util.BUFFER_OUTPUT = NWORKERS > 1
def run_one(cmd, **kwargs):
result = util.run(cmd, **kwargs)
if result:
if failfast:
sys.exit(1)
# the tests containing AssertionError might have failed because
# we spawned more workers than CPUs
# we therefore will retry them sequentially
failed[result.name] = [cmd, kwargs, 'AssertionError' in (result.output or '')]
try:
try:
for cmd, options in tests:
total += 1
spawn(run_one, cmd, **(options or {}))
gevent.wait()
except KeyboardInterrupt:
try:
if pool:
util.log('Waiting for currently running to finish...')
pool.join()
except KeyboardInterrupt:
util.report(total, failed, exit=False, took=time() - start, expected=expected)
util.log('(partial results)\n')
raise
except:
traceback.print_exc()
pool.kill() # this needed to kill the processes
raise
toretry = [key for (key, (cmd, kwargs, can_retry)) in failed.items() if can_retry]
failed_then_succeeded = []
if NWORKERS > 1 and toretry:
util.log('\nWill retry %s failed tests without concurrency:\n- %s\n', len(toretry), '\n- '.join(toretry))
for name, (cmd, kwargs, _ignore) in failed.items():
if not util.run(cmd, buffer_output=False, **kwargs):
failed.pop(name)
failed_then_succeeded.append(name)
if failed_then_succeeded:
util.log('\n%s tests failed during concurrent run but succeeded when ran sequentially:', len(failed_then_succeeded))
util.log('- ' + '\n- '.join(failed_then_succeeded))
util.log('gevent version %s from %s', gevent.__version__, gevent.__file__)
util.report(total, failed, took=time() - start, expected=expected)
assert not pool, pool
def handle(self, **options):
p = pool.Pool(20)
jobs = [
p.spawn(self.crawlblog, blog)
for blog in Blog.objects.filter(skip_crawl=False)
]
wait(jobs)
for post in self.zulip_queue:
announce_new_post(post, debug=settings.DEBUG)
def test_main_no_args():
with capture() as f:
greenlet = gevent.spawn(main, [])
gevent.sleep(0.1)
assert greenlet.started, f
greenlet.kill()
gevent.wait([greenlet])
assert not greenlet.started
def start(self):
i_count = 1
while self.have_content:
self.pool.map(self.get_one_page,
[x for x in range(i_count, i_count + 50)])
gevent.wait()
i_count += 50
print(self.bugs)
file_handle = open('wy_no_1.csv', 'w')
file_handle.write('bug_id, rank, name, corp, date\n')
for one_bug in self.bugs:
file_handle.write('{0}, {1}, {2}, {3}, {4}\n'.format(
one_bug['id'], one_bug['rank'], one_bug['name'],
one_bug['corp'], one_bug['date']))
print(one_bug)
file_handle.close()
def run(self):
self.logger.info("Firing up workers")
for i in xrange(self.config['workers']):
worker = Worker(self, i)
self.workers.append(worker)
worker.start()
self.logger.info("Starting zmq puller")
self.puller = Puller(self)
self.puller.start()
gevent.signal(signal.SIGINT, self.exit, "SIGINT")
gevent.signal(signal.SIGHUP, self.exit, "SIGHUP")
self._exit_signal = Event()
self._exit_signal.wait()
# stop all greenlets
for gl in self.workers:
self.logger.info("Requesting stop for {} greenlet".format(gl))
gl.kill(timeout=self.config['term_timeout'], block=False)
self.logger.info("Requesting stop for puller")
self.puller.kill(timeout=self.config['term_timeout'], block=False)
try:
if gevent.wait(timeout=self.config['term_timeout']):
self.logger.info("All threads exited normally")
else:
self.logger.info("Timeout reached, shutting down forcefully")
except KeyboardInterrupt:
self.logger.info("Shutdown requested again by system, "
"exiting without cleanup")
self.logger.info("=" * 80)
def main(self):
self._logger.info("PyFuzz2 Server started...")
self._beacon_server.start_server()
self._beacon_worker.start_worker()
self._report_server.start_server()
self._report_worker.start_worker()
self._web_server.start_server()
self._node_client_worker.start_worker()
self._db_worker.start_worker()
self._web_worker.start_worker()
while True:
try:
gevent.wait()
except KeyboardInterrupt:
self.__shut_down()
exit(0)
def test_later(self):
# exiting because the spawned greenlet finished execution (spawn_later (=timer) variant)
x = gevent.spawn_later(SMALL_TICK, lambda: 5)
with self.runs_in_given_time(SMALL_TICK):
result = gevent.wait(timeout=10)
self.assertTrue(result)
self.assertTrue(x.dead, x)
def test_concurrent_error_recovery(self):
"""when recovering from an error in a green thread environment one thread
could have added the table while the other thread was asleep, this will
test to make sure two threads failing at the same time will both recover
correctly"""
i = self.get_interface()
s = self.get_schema()
#i.set_table(s)
for x in range(1, 3):
gevent.spawn(i.insert, s, {'foo': x, 'bar': str(x)})
gevent.wait()
q = Query()
r = list(i.get(s, q))
self.assertEqual(2, len(r))
def main():
branch = 'rawhide'
arguments = docopt(__doc__)
package_set = _shell_utils.sanitize_pkg_names(arguments['<pkg>'])
if not package_set:
package_set = parse_package_list(
'CERTBOT-ALL-PACKAGES-AND-PLUGINS.txt')
print_in_progress('pulling new changes')
pkgs_in_process = {}
for pkg_name in package_set:
pkg_path = (THIS_DIR / '..' / pkg_name).resolve()
if _git.has_uncommitted_changes(pkg_path):
error_msg = 'uncommitted changes, skipping package'
print_status_output(pkg_name, is_error=True, msg=error_msg)
continue
_git.switch_to_branch(branch, pkg_path)
result = _git.pull('origin', pkg_path, ff_only=True, gevent=True)
pkgs_in_process[result] = pkg_name
while pkgs_in_process:
done = gevent.wait(pkgs_in_process.keys(), count=1)
for finished_proc in done:
pkg_name = pkgs_in_process.pop(finished_proc)
print_status_output(f'{pkg_name} ({branch})', is_error=False)
def main(self):
"""
Main Kobbra Server method
"""
ConsoleLogger.banner()
#mus = MusServer(self.managers)
info = InfoServer(self.managers)
#mus.start()
info.start()
gevent.wait()
self.managers.Database().Close()
ConsoleLogger.log("DBG", "Exiting!")
def stop(self):
""" Try to gracefully stop the greenlet synchronously
Stop isn't expected to re-raise greenlet _run exception
(use self.greenlet.get() for that),
but it should raise any stop-time exception """
if self._stop_event.ready():
return
self._stop_event.set()
self._client.set_presence_state(UserPresence.OFFLINE.value)
self._client.stop_listener_thread(
) # stop sync_thread, wait client's greenlets
# wait own greenlets, no need to get on them, exceptions should be raised in _run()
gevent.wait(self.greenlets)
self._client.logout()
def _translate():
while True:
r = gevent.wait(objects=[pp2_], count=1)
m = r[0].read()
print('translate')
pp2_.reset('pp2_ translate reset')
p2_.write(((sid, pid), m.msg), m.imp)
def foo(ch1, ch2):
print('Foo function. Ch1:', ch1, 'Ch2:', ch2)
while True:
r = gevent.wait([ch1])
m = r[0].read()
print('Read mesage on ch1:', m)
ch2.write(m.msg, m.imp)
def run(self):
while True:
ready = gevent.wait(
objects=[self.z2a, self.f2a, self.p2a, self._a2z, self._p2z],
count=1)
r = ready[0]
if r == self.z2a:
m = self.z2a.read()
self.z2a.reset()
self.env_msg(m)
#self._z2a.write( m )
elif r == self.f2a:
m = self.f2a.read()
self.f2a.reset()
self.func_msg(m)
elif r == self.p2a:
self.p2a.reset()
dump.dump()
elif r == self._a2z:
m = self._a2z.read()
self._a2z.reset()
self.a2z.write(m)
elif r == self._p2z:
m = self._p2z.read()
self._p2z.reset()
dump.dump()
else:
dump.dump()
def test_main(self):
def main_task():
with self.tracer.start_active_span('parent', True):
tasks = self.submit_callbacks()
gevent.joinall(tasks)
gevent.spawn(main_task)
gevent.wait(timeout=5.0)
spans = self.tracer.finished_spans()
self.assertEquals(len(spans), 4)
self.assertNamesEqual(spans, ['task', 'task', 'task', 'parent'])
for i in range(3):
self.assertSameTrace(spans[i], spans[-1])
self.assertIsChildOf(spans[i], spans[-1])
def restart_network(raiden_network, retry_timeout):
for app in raiden_network:
app.stop()
wait_network = [gevent.spawn(restart_app, app) for app in raiden_network]
gevent.wait(wait_network)
new_network = [greenlet.get() for greenlet in wait_network]
# The tests assume the nodes are available to transfer
for app0, app1 in combinations(new_network, 2):
waiting.wait_for_healthy(app0.raiden, app1.raiden.address,
retry_timeout)
return new_network
def test_event(self):
# exiting because of event (the spawned greenlet still runs)
x = gevent.spawn_later(10, lambda: 5)
event = Event()
event_set = gevent.spawn_later(SMALL_TICK, event.set)
with self.runs_in_given_time(SMALL_TICK):
result = gevent.wait([event])
self.assertEqual(result, [event])
self.assertFalse(x.dead, x)
self.assertTrue(event_set.dead)
self.assertTrue(event.is_set)
x.kill()
with self.runs_in_no_time():
result = gevent.wait()
self.assertTrue(result)
def open_and_wait_for_channels(app_channels, registry_address, token, deposit,
settle_timeout):
greenlets = []
for first_app, second_app in app_channels:
greenlets.append(
gevent.spawn(
payment_channel_open_and_deposit,
first_app,
second_app,
token,
deposit,
settle_timeout,
))
gevent.wait(greenlets)
wait_for_channels(app_channels, registry_address, [token], deposit)
def test_future_wait_gevent_function(self):
# The future object can be waited on with gevent functions.
self.assertEqual(monkey.is_module_patched('threading'),
self.MONKEY_PATCHED)
pool = self.pool
def fn():
gevent.sleep(0.5)
return 42
future = pool.submit(fn)
def spawned():
return 2016
spawned_greenlet = gevent.spawn(spawned)
done = gevent.wait((future, ))
self.assertEqual(list(done), [future])
self.assertTrue(spawned_greenlet.ready())
self.assertEqual(spawned_greenlet.value, 2016)
pool.kill()
del future
del pool
del self.pool
def init_kraken_instances(self):
"""
Call all kraken instances (as found in the instances dir) and store it's metadata
"""
futures = []
purge_cache_needed = False
for instance in self.instances.values():
if not instance.is_initialized:
futures.append(gevent.spawn(instance.init))
gevent.wait(futures)
for future in futures:
# we check if an instance needs the cache to be purged
if future.get():
self._clear_cache()
break
def handle(self, *args, **options):
def _handle_base(base):
base.stop()
base.destroy()
base.start()
greenlets = []
transaction.set_autocommit(False)
for base in Base.objects.filter(
executor__pid__isnull=False).select_for_update(nowait=False):
g = gevent.spawn(_handle_base, base)
greenlets.append(g)
gevent.wait(greenlets)
transaction.commit()
transaction.set_autocommit(True)
def start(self):
self.register_logger("gevent_helpers")
for comp in self.components.itervalues():
comp.manager = self
comp.counters = self.register_stat_counters(
comp, comp.one_min_stats, comp.one_sec_stats)
if comp is not self:
comp.logger = self.register_logger(comp.name)
comp.start()
# Starts the greenlet
Component.start(self)
# Start the datagram control server if it's been inited
if self.config['datagram']['enabled']:
DatagramServer.start(self, )
# This is the main thread of execution, so just continue here waiting
# for exit signals
######
# Register shutdown signals
gevent.signal(signal.SIGUSR1, self.dump_objgraph)
gevent.signal(signal.SIGHUP, exit, "SIGHUP")
gevent.signal(signal.SIGINT, exit, "SIGINT")
gevent.signal(signal.SIGTERM, exit, "SIGTERM")
try:
gevent.wait()
# Allow a force exit from multiple exit signals
finally:
self.logger.info(
"Exiting requested, allowing {} seconds for cleanup.".format(
self.config['term_timeout']))
try:
for comp in self.components.itervalues():
self.logger.debug(
"Calling stop on component {}".format(comp))
comp.stop()
if gevent.wait(timeout=self.config['term_timeout']):
self.logger.info("All threads exited normally")
else:
self.logger.info(
"Timeout reached, shutting down forcefully")
except gevent.GreenletExit:
self.logger.info("Shutdown requested again by system, "
"exiting without cleanup")
self.logger.info("Exit")
self.logger.info("=" * 80)
def main(filename):
print('Importing {}'.format(filename))
group_singular = {
'conifers': [
'conifer', 'plant', 'land plant', 'botany'],
'reptiles': [
'reptile', 'animal', 'cold blood', 'cold bloded', 'vertebrate',
'fauna'],
'turtles (non-marine)': [
'turtle', 'animal', 'non-marine', 'cold blood', 'cold bloded',
'vertebrate', 'fauna'],
'butterflies': [
'butterfly', 'animal', 'insect', 'moths and butterflies', 'fauna',
'invertebrate'],
'dragonflies': [
'dragonfly', 'animal', 'insect', 'dragonflies and damseflies',
'invertebrate', 'fauna'],
'mammals': [
'mammal', 'animal', 'warm blood', 'warm blooded', 'vertebrate',
'fauna'],
'birds': [
'bird', 'animal', 'warm blood', 'warm blooded', 'vertebrate',
'fauna'],
'amphibians': [
'amfibian', 'animal', 'vertebrate', 'fauna'],
'sphingid moths': [
'sphingid moth', 'moth', 'animal', 'insect', 'invertebrate',
'fauna', 'moths and butterflies'],
'bumblebees': [
'bumblebee', 'bee', 'bees', 'animal', 'insect', 'invertebrate'],
}
with open(filename, newline='') as f:
count = 0
# "Scientific Name","Common Name","Family","Taxonomic Group"
for row in csv.reader(f, delimiter=',', quotechar='"'):
count += 1
common = row[1]
if common == 'null':
common = row[0]
gevent.spawn(
post_species, row[0], common,
[row[2], row[3]] + group_singular[row[3].lower()])
if count >= 100:
gevent.wait()
count = 0
gevent.wait()
return 0
def start_blacklist(self, project):
"""
:param project:
:return:
"""
def __scan(file_path):
for rule in self.blacklist_rule:
flag = rule.verify(file_path)
if flag:
rule.get_info(file_path)
if project.web_url:
report = '{0}/blob/{1}/{2}#L{3}'.format(
project.web_url, project.branch, file_path,
rule.start_line)
else:
report = ''
author, author_email = get_git_author(
project.get_last_author(file_path))
vuln = Vulnerability(
task_id=project.task_id,
rule_key=rule.key,
risk_id=rule.risk_id,
title=rule.name,
file=file_path,
author=author,
author_email=author_email,
hash=project.get_last_commit(),
start_line=rule.start_line,
end_line=rule.end_line,
report=report,
code_example=rule.code_example,
engine=self.key,
)
vuln_key = hash_md5('{0}_{1}_{2}'.format(
file_path, rule.id, rule.start_line))
if vuln_key not in kb.result[project.key]:
kb.result[project.key][vuln_key] = vuln.info
project.logger.debug(
'[PluginScanner] {0}'.format(vuln))
pool = ThreadPool(project.threads)
for fpath, dirs, fs in os.walk(project.scan_path):
for f in fs:
pool.spawn(__scan, os.path.join(fpath, f))
gevent.wait()
def stop_and_wait(self):
# Stop handling incoming packets, but don't close the socket. The
# socket can only be safely closed after all outgoing tasks are stopped
self.transport.stop_accepting()
# Stop processing the outgoing queues
self.event_stop.set()
gevent.wait(self.greenlets)
# All outgoing tasks are stopped. Now it's safe to close the socket. At
# this point there might be some incoming message being processed,
# keeping the socket open is not useful for these.
self.transport.stop()
# Set all the pending results to False
for waitack in self.senthashes_to_states.values():
waitack.async_result.set(False)
def test_latency(apps, assets, num_transfers, amount):
def start_transfers(idx, curr_asset, num_transfers):
curr_app = apps[idx]
asset_manager = curr_app.raiden.get_manager_by_asset_address(
curr_asset)
all_paths = asset_manager.channelgraph.get_paths_of_length(
source=curr_app.raiden.address,
num_hops=2,
)
path = all_paths[0]
target = path[-1]
finished = gevent.event.Event()
def _transfer():
api = curr_app.raiden.api
for i in range(num_transfers):
async_result = api.transfer_async(curr_asset, amount, target)
async_result.wait()
finished.set()
gevent.spawn(_transfer)
return finished
finished_events = []
# Start all transfers
start_time = time.time()
for idx, curr_asset in enumerate(assets):
finished = start_transfers(idx, curr_asset, num_transfers)
finished_events.append(finished)
# Wait until the transfers for all assets are done
gevent.wait(finished_events)
elapsed = time.time() - start_time
completed_transfers = num_transfers * len(assets)
tps = completed_transfers / elapsed
print('Completed {} transfers. tps:{:.5} latency:{:.5} time:{:.5}s'.format(
completed_transfers,
tps,
elapsed / completed_transfers,
elapsed,
))
def run(self):
"""Starts all the registered services"""
for service in self.services:
if service.enabled:
self.servers.append(spawn(service.run))
self.logger.info("%d services launched", len(self.servers))
try:
if len(self.servers) > 0 and not self.stopped.is_set():
wait()
except KeyboardInterrupt:
self.logger.info('Stopping services')
self.stop()
raise KeyboardInterrupt
finally:
joinall(self.servers, 2)
def test_main_sys_args():
old_argv = list(sys.argv)
config = os.path.join(os.path.dirname(__file__), '..', '..', 'config',
'absearch.ini')
sys.argv = ['', config]
try:
with capture() as f:
greenlet = gevent.spawn(main)
gevent.sleep(0.1)
assert greenlet.started, f
greenlet.kill()
gevent.wait([greenlet])
assert not greenlet.started
finally:
sys.argv[:] = old_argv
def test_ref_attribute(self):
# checking "ref=False" attribute
w = gevent.get_hub().loop.timer(10)
w.start(lambda: None)
w.ref = False
with self.runs_in_no_time():
result = gevent.wait()
self.assertTrue(result)
def test_callback(self):
# exiting because the spawned greenlet finished execution (spawn (=callback) variant)
x = gevent.spawn(lambda: 5)
with self.runs_in_no_time():
result = gevent.wait(timeout=10)
self.assertTrue(result)
self.assertTrue(x.dead, x)
self.assertEqual(x.value, 5)
def test_run(self, chat):
client = Mock()
chat.pubsub = Mock()
chat.pubsub.channels = ['quorum']
chat.pubsub.listen.return_value = [{
'type': 'message',
'channel': 'quorum',
'data': 'Calloo! Callay!',
}]
chat.greenlet = Mock()
chat.subscribe(client, 'quorum')
chat.run()
gevent.wait() # wait for event loop
client.send.assert_called_once_with('quorum:Calloo! Callay!')
def work(self, burst=False):
"""Starts the work loop.
Pops and performs all jobs on the current list of queues. When all
queues are empty, block and wait for new jobs to arrive on any of the
queues, unless `burst` mode is enabled.
The return value indicates whether any jobs were processed.
"""
setup_loghandlers()
self._install_signal_handlers()
self.did_perform_work = False
self.register_birth()
self.log.info('RQ worker started, version %s' % VERSION)
self.set_state('starting')
try:
while True:
if self.stopped:
self.log.info('Stopping on request.')
break
timeout = None if burst else max(1, self.default_worker_ttl - 60)
try:
result = self.dequeue_job_and_maintain_ttl(timeout)
if result is None and burst:
try:
# Make sure dependented jobs are enqueued.
gevent.wait(self.children)
except LoopExit:
pass
result = self.dequeue_job_and_maintain_ttl(timeout)
if result is None:
break
except StopRequested:
break
job, queue = result
self.execute_job(job, queue)
finally:
if not self.is_horse:
self.register_death()
return self.did_perform_work
def test_direct_transfer_to_offline_node(raiden_network, token_addresses,
deposit):
app0, app1 = raiden_network
token_address = token_addresses[0]
chain_state = views.state_from_app(app0)
payment_network_id = app0.raiden.default_registry.address
token_network_identifier = views.get_token_network_identifier_by_token_address(
chain_state,
payment_network_id,
token_address,
)
# Wait until the initialization of the node is complete and then stop it
gevent.wait([app1.raiden.start_event])
app1.raiden.stop()
amount = 10
target = app1.raiden.address
payment_identifier = 13
app0.raiden.direct_transfer_async(
token_network_identifier,
amount,
target,
identifier=payment_identifier,
)
app1.raiden.start()
exception = ValueError(
'Waiting for transfer received success in the WAL timed out')
with gevent.Timeout(seconds=5, exception=exception):
wait_for_transfer_success(
app1.raiden,
payment_identifier,
amount,
app1.raiden.alarm.sleep_time,
)
no_outstanding_locks = []
assert_synced_channel_state(
token_network_identifier,
app0,
deposit - amount,
no_outstanding_locks,
app1,
deposit + amount,
no_outstanding_locks,
)
