def crawl(self):
"""Run the crawler until all finished."""
with (yield From(self.termination)):
while self.todo or self.busy:
if self.todo:
url, max_redirect = self.todo.popitem()
fetcher = Fetcher(
self.log,
url,
crawler=self,
max_redirect=max_redirect,
max_tries=self.max_tries,
)
self.busy[url] = fetcher
fetcher.task = asyncio.Task(self.fetch(fetcher))
else:
yield From(self.termination.wait())
self.t1 = time.time()
def test_readexactly_eof(self):
# Read exact number of bytes (eof).
stream = asyncio.StreamReader(loop=self.loop)
n = 2 * len(self.DATA)
read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)
def cb():
stream.feed_data(self.DATA)
stream.feed_eof()
self.loop.call_soon(cb)
with self.assertRaises(asyncio.IncompleteReadError) as cm:
self.loop.run_until_complete(read_task)
self.assertEqual(cm.exception.partial, self.DATA)
self.assertEqual(cm.exception.expected, n)
self.assertEqual(str(cm.exception),
'18 bytes read on a total of 36 expected bytes')
self.assertEqual(b'', stream._buffer)
def inferiortty(self, set_inferior_tty_cb):
"""Spawn the inferior terminal."""
@asyncio.coroutine
def _set_inferior_tty():
if self.proc_inftty:
if self.proc_inftty.returncode is None:
self.proc_inftty.terminate()
self.proc_inftty = None
try:
self.proc_inftty = proc = yield asyncio.From(
asyncio.create_subprocess_exec(*args))
info('inferiortty: {}'.format(args))
except OSError as e:
self.console_print('Cannot spawn terminal: {}\n'.format(e))
else:
start = time.time()
while time.time() - start < 2:
try:
with open(result_file.name) as f:
lines = f.readlines()
# Commands found in the result file.
if len(lines) == 2:
set_inferior_tty_cb(lines[0])
set_inferior_tty_cb(lines[1])
break
except IOError as e:
self.console_print(
'Cannot set the inferior tty: {}\n'.format(e))
proc.terminate()
break
yield asyncio.From(asyncio.sleep(.100, loop=self.vim.loop))
else:
self.console_print('Failed to start inferior_tty.py.\n')
proc.terminate()
args = self.vim.options.terminal.split(',')
result_file = misc.tmpfile('dbg')
args.append('%s -m clewn.inferiortty %s' %
(sys.executable, result_file.name))
asyncio.Task(_set_inferior_tty(), loop=self.vim.loop)
def run():
connection = yield From(
trollius_redis.Pool.create(host=u'localhost',
port=6379,
poolsize=50))
try:
# === Benchmark 1 ==
print(u'1. How much time does it take to set 10,000 values '
u'in Redis? (without pipelining)')
print(u'Starting...')
start = time.time()
# Do 10,000 set requests
for i in range(10 * 1000):
# By using yield from here, we wait for the answer.
yield From(connection.set(u'key', u'value'))
print(u'Done. Duration=', time.time() - start)
print()
# === Benchmark 2 (should be at least 3x as fast) ==
print(u'2. How much time does it take if we use asyncio.gather, '
u'and pipeline requests?')
print(u'Starting...')
start = time.time()
# Do 10,000 set requests
futures = [
asyncio.Task(connection.set(u'key', u'value'))
for x in range(10 * 1000)
]
yield From(asyncio.gather(*futures))
print(u'Done. Duration=', time.time() - start)
finally:
connection.close()
def test_wait_for(self):
cond = asyncio.Condition(loop=self.loop)
presult = False
def predicate():
return presult
result = []
@asyncio.coroutine
def c1(result):
yield From(cond.acquire())
if (yield From(cond.wait_for(predicate))):
result.append(1)
cond.release()
raise Return(True)
t = asyncio.Task(c1(result), loop=self.loop)
test_utils.run_briefly(self.loop)
self.assertEqual([], result)
self.loop.run_until_complete(cond.acquire())
cond.notify()
cond.release()
test_utils.run_briefly(self.loop)
self.assertEqual([], result)
presult = True
self.loop.run_until_complete(cond.acquire())
cond.notify()
cond.release()
test_utils.run_briefly(self.loop)
self.assertEqual([1], result)
self.assertTrue(t.done())
self.assertTrue(t.result())
def main(testrun=False):
"""Main.
Return the vim instance to avoid its 'f_script' member to be garbage
collected and the corresponding 'TmpFile' to be unlinked before Vim has a
chance to start and source the file (only needed for the pdb test suite).
"""
vim = Vim(testrun, sys.argv[1:])
options = vim.options
is_pdb = (vim.module == 'pdb')
try:
# Vim is running the command ':Pyclewn pdb script'.
if is_pdb and not testrun and options.args:
embed_pdb(vim)
return vim
vim.set_event_loop()
vim.setup()
# When is_pdb is True, this is either:
# * Vim is being spawned by the test suite.
# * or the user is running the Vim command ':Pyclewn pdb'
if is_pdb:
if not testrun and not options.cargs:
critical('This command is meant to be run from Vim.')
return
vim_tasks = []
# Use pyclewn terminal as the inferior standard input/output when vim is
# run as 'gvim'.
if (options.editor and vim.module == 'gdb' and not testrun
and os.isatty(sys.stdin.fileno()) and not options.daemon):
out = exec_vimcmd(['echo has("gui_running")'], options.editor,
vim.stderr_hdlr).strip()
if out == '1':
tasks, ptyname = tty.inferior_tty(vim.stderr_hdlr, vim.loop)
options.tty = ptyname
vim_tasks.extend(tasks)
vim_tasks.append(asyncio.Task(vim.run(), loop=vim.loop))
misc.cancel_after_first_completed(vim_tasks,
lambda: vim.signal(None),
loop=vim.loop)
except Exception as e:
except_str = 'Exception in pyclewn: "%s"\n' \
'%s\n' \
'pyclewn aborting...\n' \
% (e, traceback.format_tb(sys.exc_info()[2])[-1])
critical('\n' + except_str)
if vim.netbeans:
vim.netbeans.show_balloon(except_str)
if not testrun:
sys.exit(1)
finally:
debug('Vim instance: ' + str(vim))
vim.shutdown()
return vim
def getaddrinfo_task(*args, **kwds):
return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
"""
import trollius
from trollius import From
@trollius.coroutine
def factorial(name, number):
"""
https://en.wikipedia.org/wiki/Factorial
"""
f = 1
for i in range(2, number + 1):
print("Task %s: Compute factorial(%s)..." % (name, i))
yield trollius.sleep(1)
f *= i
print("Task %s: factorial(%s) = %s" % (name, number, f))
# Instantiating tasks doesn't cause the coroutine to be run. It merely
# schedules the tasks.
tasks = [
trollius.Task(factorial("A", 2)),
trollius.Task(factorial("B", 3)),
trollius.Task(factorial("C", 4)),
]
# Get the event loop and cause it to run until all the tasks are done.
loop = trollius.get_event_loop()
loop.run_until_complete(trollius.wait(tasks))
loop.close()
def test_put_with_waiting_getters(self):
q = asyncio.Queue(loop=self.loop)
t = asyncio.Task(q.get(), loop=self.loop)
test_utils.run_briefly(self.loop)
self.loop.run_until_complete(q.put('a'))
self.assertEqual(self.loop.run_until_complete(t), 'a')
def test_acquire(self):
sem = asyncio.Semaphore(3, loop=self.loop)
result = []
self.assertTrue(self.loop.run_until_complete(sem.acquire()))
self.assertTrue(self.loop.run_until_complete(sem.acquire()))
self.assertFalse(sem.locked())
@asyncio.coroutine
def c1(result):
yield From(sem.acquire())
result.append(1)
raise Return(True)
@asyncio.coroutine
def c2(result):
yield From(sem.acquire())
result.append(2)
raise Return(True)
@asyncio.coroutine
def c3(result):
yield From(sem.acquire())
result.append(3)
raise Return(True)
@asyncio.coroutine
def c4(result):
yield From(sem.acquire())
result.append(4)
raise Return(True)
t1 = asyncio.Task(c1(result), loop=self.loop)
t2 = asyncio.Task(c2(result), loop=self.loop)
t3 = asyncio.Task(c3(result), loop=self.loop)
# each coroutine requires 2 runs of the event loop
test_utils.run_briefly(self.loop, 2)
self.assertEqual([1], result)
self.assertTrue(sem.locked())
self.assertEqual(2, len(sem._waiters))
self.assertEqual(0, sem._value)
t4 = asyncio.Task(c4(result), loop=self.loop)
sem.release()
sem.release()
self.assertEqual(2, sem._value)
test_utils.run_briefly(self.loop)
self.assertEqual(0, sem._value)
self.assertEqual([1, 2, 3], result)
self.assertTrue(sem.locked())
self.assertEqual(1, len(sem._waiters))
self.assertEqual(0, sem._value)
self.assertTrue(t1.done())
self.assertTrue(t1.result())
self.assertTrue(t2.done())
self.assertTrue(t2.result())
self.assertTrue(t3.done())
self.assertTrue(t3.result())
self.assertFalse(t4.done())
# cleanup locked semaphore
sem.release()
self.loop.run_until_complete(t4)
def test_wait(self):
cond = asyncio.Condition(loop=self.loop)
result = []
@asyncio.coroutine
def c1(result):
yield From(cond.acquire())
if (yield From(cond.wait())):
result.append(1)
raise Return(True)
@asyncio.coroutine
def c2(result):
yield From(cond.acquire())
if (yield From(cond.wait())):
result.append(2)
raise Return(True)
@asyncio.coroutine
def c3(result):
yield From(cond.acquire())
if (yield From(cond.wait())):
result.append(3)
raise Return(True)
t1 = asyncio.Task(c1(result), loop=self.loop)
t2 = asyncio.Task(c2(result), loop=self.loop)
t3 = asyncio.Task(c3(result), loop=self.loop)
test_utils.run_briefly(self.loop, 2)
self.assertEqual([], result)
self.assertFalse(cond.locked())
self.assertTrue(self.loop.run_until_complete(cond.acquire()))
cond.notify()
test_utils.run_briefly(self.loop, 2)
self.assertEqual([], result)
self.assertTrue(cond.locked())
cond.release()
test_utils.run_briefly(self.loop, 2)
self.assertEqual([1], result)
self.assertTrue(cond.locked())
cond.notify(2)
test_utils.run_briefly(self.loop, 2)
self.assertEqual([1], result)
self.assertTrue(cond.locked())
cond.release()
test_utils.run_briefly(self.loop, 2)
self.assertEqual([1, 2], result)
self.assertTrue(cond.locked())
cond.release()
test_utils.run_briefly(self.loop, 2)
self.assertEqual([1, 2, 3], result)
self.assertTrue(cond.locked())
self.assertTrue(t1.done())
self.assertTrue(t1.result())
self.assertTrue(t2.done())
self.assertTrue(t2.result())
self.assertTrue(t3.done())
self.assertTrue(t3.result())
def test_start_server(self):
class MyServer:
def __init__(self, loop):
self.server = None
self.loop = loop
@asyncio.coroutine
def handle_client(self, client_reader, client_writer):
data = yield From(client_reader.readline())
client_writer.write(data)
yield From(client_writer.drain())
client_writer.close()
def start(self):
sock = socket.socket()
sock.bind(('127.0.0.1', 0))
self.server = self.loop.run_until_complete(
asyncio.start_server(self.handle_client,
sock=sock,
loop=self.loop))
return sock.getsockname()
def handle_client_callback(self, client_reader, client_writer):
self.loop.create_task(
self.handle_client(client_reader, client_writer))
def start_callback(self):
sock = socket.socket()
sock.bind(('127.0.0.1', 0))
addr = sock.getsockname()
sock.close()
self.server = self.loop.run_until_complete(
asyncio.start_server(self.handle_client_callback,
host=addr[0],
port=addr[1],
loop=self.loop))
return addr
def stop(self):
if self.server is not None:
self.server.close()
self.loop.run_until_complete(self.server.wait_closed())
self.server = None
@asyncio.coroutine
def client(addr):
reader, writer = yield From(
asyncio.open_connection(*addr, loop=self.loop))
# send a line
writer.write(b"hello world!\n")
# read it back
msgback = yield From(reader.readline())
writer.close()
raise Return(msgback)
# test the server variant with a coroutine as client handler
server = MyServer(self.loop)
addr = server.start()
msg = self.loop.run_until_complete(
asyncio.Task(client(addr), loop=self.loop))
server.stop()
self.assertEqual(msg, b"hello world!\n")
# test the server variant with a callback as client handler
server = MyServer(self.loop)
addr = server.start_callback()
msg = self.loop.run_until_complete(
asyncio.Task(client(addr), loop=self.loop))
server.stop()
self.assertEqual(msg, b"hello world!\n")
def start_other():
asyncio.Task(second_coroutine())
def start_client(loop, host, port):
t = asyncio.Task(loop.create_connection(EchoClient, host, port))
loop.run_until_complete(t)
def CriticalTask(coro):
task = asyncio.Task(coro)
task.add_done_callback(_critical_task_done)
return task
def send_message(self, msg):
self._producers.append(
asyncio.Task(self._queue.put(msg), loop=self._loop), )
if sorted(res) != res:
print('FAIL', res)
print('------------')
else:
print('.', end='')
sys.stdout.flush()
loop = asyncio.get_event_loop()
print('Pass 1')
# All permutations of discrete task running times must be returned
# by as_completed in the correct order.
task_times = [0, 0.1, 0.2, 0.3, 0.4] # 120 permutations
for times in itertools.permutations(task_times):
tasks = [asyncio.Task(sleeper(t)) for t in times]
loop.run_until_complete(asyncio.Task(watcher(tasks)))
print()
print('Pass 2')
# Longer task times, with randomized duplicates. 100 tasks each time.
longer_task_times = [x / 10 for x in range(30)]
for i in range(20):
task_times = longer_task_times * 10
random.shuffle(task_times)
#print('Times', task_times[:500])
tasks = [asyncio.Task(sleeper(t)) for t in task_times[:100]]
loop.run_until_complete(asyncio.Task(watcher(tasks)))
print()
print('Pass 3')
def test_start_unix_server(self):
class MyServer:
def __init__(self, loop, path):
self.server = None
self.loop = loop
self.path = path
@asyncio.coroutine
def handle_client(self, client_reader, client_writer):
data = yield From(client_reader.readline())
client_writer.write(data)
yield From(client_writer.drain())
client_writer.close()
def start(self):
self.server = self.loop.run_until_complete(
asyncio.start_unix_server(self.handle_client,
path=self.path,
loop=self.loop))
def handle_client_callback(self, client_reader, client_writer):
self.loop.create_task(
self.handle_client(client_reader, client_writer))
def start_callback(self):
start = asyncio.start_unix_server(self.handle_client_callback,
path=self.path,
loop=self.loop)
self.server = self.loop.run_until_complete(start)
def stop(self):
if self.server is not None:
self.server.close()
self.loop.run_until_complete(self.server.wait_closed())
self.server = None
@asyncio.coroutine
def client(path):
reader, writer = yield From(
asyncio.open_unix_connection(path, loop=self.loop))
# send a line
writer.write(b"hello world!\n")
# read it back
msgback = yield From(reader.readline())
writer.close()
raise Return(msgback)
# test the server variant with a coroutine as client handler
with test_utils.unix_socket_path() as path:
server = MyServer(self.loop, path)
server.start()
msg = self.loop.run_until_complete(
asyncio.Task(client(path), loop=self.loop))
server.stop()
self.assertEqual(msg, b"hello world!\n")
# test the server variant with a callback as client handler
with test_utils.unix_socket_path() as path:
server = MyServer(self.loop, path)
server.start_callback()
msg = self.loop.run_until_complete(
asyncio.Task(client(path), loop=self.loop))
server.stop()
self.assertEqual(msg, b"hello world!\n")
[[raw_name_matrix[index][0], raw_name_matrix[index][1]],
[in_cof_matrix[0], in_cof_matrix[1]]])
from multiprocessing.pool import ThreadPool
tasks = []
final_f = []
final_combination = []
for index, value in enumerate(score_matrix):
new_data = result_dta_numenta.copy()
new_data.anomaly_score = value
#engine.anomaly_detection(new_data, raw_dta, name_coff_metrix[index]);
tasks.append(
asyncio.Task(
engine.anomaly_detection_asysn(new_data, raw_dta,
name_coff_metrix[index])))
#final_f.append(return_val)
final_combination.append(name_coff_metrix[index])
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
end = time.time()
print("Total time: {}".format(end - start))
# print final_f
# final_index = np.argsort(final_f)[-1]
# filed_name = final_combination[final_index]
#
# print "%%%%%%%%%%%%%%%%%%%%%%%%______BEST CHOICE______%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%"
raise Return(cropped)
if __name__ == '__main__':
# Build context
CASCADES_DIR = '/usr/share/opencv/haarcascades'
context = Context(wname='Video',
faceCascade=cv2.CascadeClassifier(
os.path.join(CASCADES_DIR,
'haarcascade_frontalface_default.xml')),
eyeCascade=cv2.CascadeClassifier(
os.path.join(CASCADES_DIR, 'haarcascade_eye.xml')))
# Initialize pipeline
tasks = [
asyncio.Task(webcam(context)),
asyncio.Task(compose(context)),
asyncio.Task(display(context))
]
# This will suspend the main routine until q is pressed
loop = asyncio.get_event_loop()
loop.run_until_complete(keyboard(context))
for t in tasks:
t.cancel()
# Give tasks a chance to shut down gracefully
loop.run_until_complete(asyncio.wait(tasks))
def start_server(loop, addr):
import_sdl2()
t = asyncio.Task(
loop.create_datagram_endpoint(ServerProtocol, local_addr=addr))
transport, server = loop.run_until_complete(t)
return transport
def test():
get_task = asyncio.Task(queue_get(), loop=loop)
yield From(asyncio.sleep(0.01, loop=loop)) # let the task start
q.put_nowait(1)
result = (yield From(get_task))
raise Return(result)
def start_client(loop, addr):
t = asyncio.Task(
loop.create_datagram_endpoint(ClientProtocol, remote_addr=addr))
loop.run_until_complete(t)
message = pygazebo.msg.pose_pb2.Pose()
message.position.x = U.translate_speed
message.orientation.z = rpy_to_quaternion(
Rpy(0, 0, (degree_to_rad(U.rotate_speed)))).z
while True:
yield From(publisher.publish(message))
yield From(trollius.sleep(0.1))
@trollius.coroutine
def server_loop(U):
U.server_run()
if __name__ == '__main__':
util = PyTango.Util(sys.argv)
util.add_class(PyDevice, PyTrevorGazebo)
U = PyTango.Util.instance()
U.server_init()
tasks = [
trollius.Task(publish_loop(U)),
trollius.Task(server_loop(U)),
]
loop = trollius.get_event_loop()
loop.run_until_complete(trollius.wait(tasks))
loop.close()
def main(loop):
# program which prints evaluation of each expression from stdin
code = r'''if 1:
import os
def writeall(fd, buf):
while buf:
n = os.write(fd, buf)
buf = buf[n:]
while True:
s = os.read(0, 1024)
if not s:
break
s = s.decode('ascii')
s = repr(eval(s)) + '\n'
s = s.encode('ascii')
writeall(1, s)
'''
# commands to send to input
commands = iter([b"1+1\n", b"2**16\n", b"1/3\n", b"'x'*50", b"1/0\n"])
# start subprocess and wrap stdin, stdout, stderr
p = Popen([sys.executable, '-c', code],
stdin=PIPE,
stdout=PIPE,
stderr=PIPE)
stdin = yield From(connect_write_pipe(p.stdin))
stdout, stdout_transport = yield From(connect_read_pipe(p.stdout))
stderr, stderr_transport = yield From(connect_read_pipe(p.stderr))
# interact with subprocess
name = {stdout: 'OUT', stderr: 'ERR'}
registered = {
asyncio.Task(stderr.readline()): stderr,
asyncio.Task(stdout.readline()): stdout
}
while registered:
# write command
cmd = next(commands, None)
if cmd is None:
stdin.close()
else:
print('>>>', cmd.decode('ascii').rstrip())
stdin.write(cmd)
# get and print lines from stdout, stderr
timeout = None
while registered:
done, pending = yield From(
asyncio.wait(registered,
timeout=timeout,
return_when=asyncio.FIRST_COMPLETED))
if not done:
break
for f in done:
stream = registered.pop(f)
res = f.result()
print(name[stream], res.decode('ascii').rstrip())
if res != b'':
registered[asyncio.Task(stream.readline())] = stream
timeout = 0.0
stdout_transport.close()
stderr_transport.close()
