def test_wait_concurrent_complete(self):
def gen():
when = yield
self.assertAlmostEqual(0.1, when)
when = yield 0
self.assertAlmostEqual(0.15, when)
when = yield 0
self.assertAlmostEqual(0.1, when)
yield 0.1
loop = test_utils.TestLoop(gen)
self.addCleanup(loop.close)
a = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop)
b = asyncio.Task(asyncio.sleep(0.15, loop=loop), loop=loop)
done, pending = loop.run_until_complete(
asyncio.wait([b, a], timeout=0.1, loop=loop))
self.assertEqual(done, set([a]))
self.assertEqual(pending, set([b]))
self.assertAlmostEqual(0.1, loop.time())
# move forward to close generator
loop.advance_time(10)
loop.run_until_complete(asyncio.wait([a, b], loop=loop))
def scrap(category, field="", pages=[1], search="", use_proxy=False, proxy_file=None, use_tor=[]):
"""
Will scrap all links from given pages (list).
field = "time_add" for "AGE" ordering, "seeders" for "SEED".
search = "movies", "tv" or blank.
"""
conngen = connector_gen(use_proxy, proxy_file, use_tor)
if use_tor[0]: kat_site = "http://lsuzvpko6w6hzpnn.onion"
else: kat_site = choice(KATCR)
if search: search += " " # Otherwise invalid link
urls = [ "{}/usearch/{}category:{}/{}/".format(kat_site, search, category, page)
for page in pages
if page > 0 ]
print(urls)
params = {"field": field, "sorder": "desc"}
loop = asyncio.get_event_loop()
# Scrap links for torrents
begin = perf_counter()
to_do = [ get_pages(url, params, conngen) for url in urls ]
wait_coro = asyncio.wait(to_do)
res, _ = loop.run_until_complete(wait_coro)
links = []
for html in res:
bs = Soup(html.result(), PARSER)
torrents = bs.find_all("tr", {"id": re.compile("torrent_[A-Za-z0-9_]*")})
links += [ choice(KATCR) + row.find("a", {"class": "cellMainLink"}).get("href")
for row in torrents ]
delay(begin, "get torrents from pages.")
# Scrap info from kickass
begin = perf_counter()
to_do = [ load_url(link, conngen) for link in links ]
wait_coro = asyncio.wait(to_do)
res, _ = loop.run_until_complete(wait_coro)
delay(begin, "load_url().")
# Scrap info from OMDB API
begin = perf_counter()
to_do = [ get_omdb(row.result(), conngen) for row in res
if row.result() is not None ]
wait_coro = asyncio.wait(to_do)
res, _ = loop.run_until_complete(wait_coro)
delay(begin, "get_omdb().")
# Download images
begin = perf_counter()
to_do = [ down_images(row.result(), conngen) for row in res ]
wait_coro = asyncio.wait(to_do)
res, _ = loop.run_until_complete(wait_coro)
delay(begin, "down_images().")
# loop.close()
# Return list of dictionaries about torrents
return [ tor.result() for tor in res ]
def web_sockets_handler(self, websocket, path):
self.__connected.add(websocket)
print('Connected: %s' % websocket)
try:
yield from asyncio.wait([ws.send('client connected') for ws in self.__connected])
while True:
listener_task = asyncio.ensure_future(websocket.recv())
producer_task = asyncio.ensure_future(self.producer())
done, pending = yield from asyncio.wait(
[listener_task, producer_task],
return_when=asyncio.FIRST_COMPLETED)
if listener_task in done:
message = listener_task.result()
yield from self.consumer(message)
else:
listener_task.cancel()
if producer_task in done:
message = producer_task.result()
yield from websocket.send(message)
else:
producer_task.cancel()
except websockets.exceptions.ConnectionClosed:
print('Connection %s closed' % websocket)
finally:
self.__connected.remove(websocket)
def test_wait_first_completed(self):
def gen():
when = yield
self.assertAlmostEqual(10.0, when)
when = yield 0
self.assertAlmostEqual(0.1, when)
yield 0.1
loop = test_utils.TestLoop(gen)
self.addCleanup(loop.close)
a = asyncio.Task(asyncio.sleep(10.0, loop=loop), loop=loop)
b = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop)
task = asyncio.Task(
asyncio.wait([b, a], return_when=asyncio.FIRST_COMPLETED,
loop=loop),
loop=loop)
done, pending = loop.run_until_complete(task)
self.assertEqual({b}, done)
self.assertEqual({a}, pending)
self.assertFalse(a.done())
self.assertTrue(b.done())
self.assertIsNone(b.result())
self.assertAlmostEqual(0.1, loop.time())
# move forward to close generator
loop.advance_time(10)
loop.run_until_complete(asyncio.wait([a, b], loop=loop))
def test_wait_first_exception_in_wait(self):
def gen():
when = yield
self.assertAlmostEqual(10.0, when)
when = yield 0
self.assertAlmostEqual(0.01, when)
yield 0.01
loop = test_utils.TestLoop(gen)
self.addCleanup(loop.close)
# first_exception, exception during waiting
a = asyncio.Task(asyncio.sleep(10.0, loop=loop), loop=loop)
@asyncio.coroutine
def exc():
yield from asyncio.sleep(0.01, loop=loop)
raise ZeroDivisionError('err')
b = asyncio.Task(exc(), loop=loop)
task = asyncio.wait([b, a], return_when=asyncio.FIRST_EXCEPTION,
loop=loop)
done, pending = loop.run_until_complete(task)
self.assertEqual({b}, done)
self.assertEqual({a}, pending)
self.assertAlmostEqual(0.01, loop.time())
# move forward to close generator
loop.advance_time(10)
loop.run_until_complete(asyncio.wait([a, b], loop=loop))
def run(self, proxy):
# There is no point in running more workers than we have URLs
# (left) to process.
nworkers = min(self.max_workers, len(self.urls))
# Unlike wait_for(), wait() does _not_ cancel everything it's
# waiting for when it is itself cancelled. Since that's what
# we want, we have to do it by hand.
try:
workers = [self.loop.create_task(self.run_worker(proxy, i))
for i in range(nworkers)]
done, pending = yield from asyncio.wait(workers, loop=self.loop)
except:
for w in workers: w.cancel()
yield from asyncio.wait(workers, loop=self.loop)
raise
# Detect and propagate any failures
assert len(pending) == 0
for w in done:
w.result()
# If we get here, we should be completely done with this locale.
assert len(self.urls) == 0
proxy.close()
def async_service_handle(service):
"""Handle service ffmpeg process."""
entity_ids = service.data.get(ATTR_ENTITY_ID)
if entity_ids:
devices = [device for device in manager.entities
if device.entity_id in entity_ids]
else:
devices = manager.entities
tasks = []
for device in devices:
if service.service == SERVICE_START:
tasks.append(device.async_start_ffmpeg())
elif service.service == SERVICE_STOP:
tasks.append(device.async_stop_ffmpeg())
else:
tasks.append(device.async_restart_ffmpeg())
if tasks:
yield from asyncio.wait(tasks, loop=hass.loop)
tasks.clear()
for device in devices:
tasks.append(device.async_update_ha_state())
if tasks:
yield from asyncio.wait(tasks, loop=hass.loop)
def test_all_on_paired_peers(self):
conns = connection_pair(self.loop)
rpc = RPC("json", self.loop)
@asyncio.coroutine
def echo_tag(ch):
obj, _ = yield from ch.recv()
obj["tag"] = True
yield from ch.send(obj)
rpc.register("echo-tag", echo_tag)
tasks = [
self.spawn(rpc.accept(conns[0], False)),
self.spawn(rpc.handshake(conns[1], False)),
]
yield from asyncio.wait(tasks)
peer1 = tasks[0].result()
peer2 = tasks[1].result()
tasks = [
self.spawn(peer1.call("echo-tag", {"from": "peer1"})),
self.spawn(peer2.call("echo-tag", {"from": "peer2"})),
peer1.route(2),
peer2.route(2),
]
yield from asyncio.wait(tasks + peer1.tasks + peer2.tasks)
yield from conns[0].close()
yield from conns[1].close()
self.assertEqual(tasks[0].result()["from"], "peer1")
self.assertEqual(tasks[0].result()["tag"], True)
self.assertEqual(tasks[1].result()["from"], "peer2")
self.assertEqual(tasks[1].result()["tag"], True)
def broadcast_application_message(self, source_session, topic, data, force_qos=None):
#self.logger.debug("Broadcasting message from %s on topic %s" %
# (format_client_message(session=source_session), topic)
# )
publish_tasks = []
try:
for k_filter in self._subscriptions:
if self.matches(topic, k_filter):
subscriptions = self._subscriptions[k_filter]
for subscription in subscriptions:
target_session = subscription.session
qos = subscription.qos
if force_qos is not None:
qos = force_qos
if target_session.transitions.state == 'connected':
self.logger.debug("broadcasting application message from %s on topic '%s' to %s" %
(format_client_message(session=source_session),
topic, format_client_message(session=target_session)))
handler = subscription.session.handler
publish_tasks.append(
asyncio.Task(handler.mqtt_publish(topic, data, qos, retain=False))
)
else:
self.logger.debug("retaining application message from %s on topic '%s' to client '%s'" %
(format_client_message(session=source_session),
topic, format_client_message(session=target_session)))
retained_message = RetainedApplicationMessage(source_session, topic, data, qos)
publish_tasks.append(
asyncio.Task(target_session.retained_messages.put(retained_message))
)
if len(publish_tasks) > 0:
asyncio.wait(publish_tasks)
except Exception as e:
self.logger.warn("Message broadcasting failed: %s", e)
def suspend_post(self):
'''
Method called after host system wake up from sleep.
:return:
'''
coros = []
# first resume/unpause VMs
for vm in self.app.domains:
if isinstance(vm, qubes.vm.adminvm.AdminVM):
continue
if vm.get_power_state() in ["Paused", "Suspended"]:
coros.append(vm.resume())
if coros:
yield from asyncio.wait(coros)
# then notify all VMs
processes = []
for vm in self.app.domains:
if isinstance(vm, qubes.vm.adminvm.AdminVM):
continue
if vm.is_running():
proc = yield from vm.run_service(
'qubes.SuspendPostAll', user='******',
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
processes.append(proc)
# FIXME: some timeout?
if processes:
yield from asyncio.wait([p.wait() for p in processes])
def suspend_pre(self):
'''
Method called before host system goes to sleep.
:return:
'''
# first notify all VMs
processes = []
for vm in self.app.domains:
if isinstance(vm, qubes.vm.adminvm.AdminVM):
continue
if vm.is_running():
proc = yield from vm.run_service(
'qubes.SuspendPreAll', user='******',
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
processes.append(proc)
# FIXME: some timeout?
if processes:
yield from asyncio.wait([p.wait() for p in processes])
coros = []
# then suspend/pause VMs
for vm in self.app.domains:
if isinstance(vm, qubes.vm.adminvm.AdminVM):
continue
if vm.is_running():
coros.append(vm.suspend())
if coros:
yield from asyncio.wait(coros)
def test_class_method_coro(self):
# test that our exception is being raised
class Test:
count = 0
@classmethod
@asyncio.coroutine
def method(cls, *args, **kwargs):
cls.count += 1
callback = Test.method
signal = Signal(loop=self.loop)
tasks = [self.loop.create_task(signal.connect(callback))]
self.loop.run_until_complete(asyncio.wait(tasks))
for task in tasks:
task.result()
tasks = [self.loop.create_task(signal.send())]
self.loop.run_until_complete(asyncio.wait(tasks))
for task in tasks:
task.result()
self.assertEqual(Test.count, 1)
def test_weakref_all_no_args_multiple(self):
fn1 = FunctionMock()
fn2 = FunctionMock()
fn3 = FunctionMock()
fn4 = FunctionMock()
signal = Signal(loop=self.loop)
tasks = [self.loop.create_task(signal.connect(fn1)),
self.loop.create_task(signal.connect(fn2)),
self.loop.create_task(signal.connect(fn3)),
self.loop.create_task(signal.connect(fn4)),
self.loop.create_task(signal.send())]
self.loop.run_until_complete(asyncio.wait(tasks))
self.assertEqual(len(signal._all), 4)
del(fn1)
gc.collect()
# cleanup happens during disconnect and send only.
tasks = [self.loop.create_task(signal.send())]
self.loop.run_until_complete(asyncio.wait(tasks))
self.assertEqual(len(signal._all), 3)
def test_exception_coro(self):
# test that our exception is being raised
exception_handler = Mock()
self.loop.set_exception_handler(exception_handler)
callback_coro = CoroutineMock()
callback_coro.side_effect = Exception('BOOM!')
signal = Signal(loop=self.loop)
tasks = [self.loop.create_task(signal.connect(callback_coro))]
self.loop.run_until_complete(asyncio.wait(tasks))
for task in tasks:
task.result()
tasks = [self.loop.create_task(signal.send())]
self.loop.run_until_complete(asyncio.wait(tasks))
for task in tasks:
task.result()
self.assertTrue(callback_coro.called)
self.assertTrue(exception_handler.called)
# reset our exception handler
self.loop.set_exception_handler(None)
def test_static_method(self):
# test that our exception is being raised
count = 0
class Test:
@staticmethod
def method(*args, **kwargs):
nonlocal count
count += 1
callback = Test.method
signal = Signal(loop=self.loop)
tasks = [self.loop.create_task(signal.connect(callback))]
self.loop.run_until_complete(asyncio.wait(tasks))
for task in tasks:
task.result()
tasks = [self.loop.create_task(signal.send())]
self.loop.run_until_complete(asyncio.wait(tasks))
for task in tasks:
task.result()
self.assertEqual(count, 1)
def test_weakref_keys(self):
callback = FunctionMock()
key = 'key1'
key2 = 'key2'
signal = Signal(loop=self.loop)
tasks = [self.loop.create_task(signal.connect(callback, keys=[key, key2]))]
self.loop.run_until_complete(asyncio.wait(tasks))
for task in tasks:
task.result()
# delete and do garbage cleanup
del(callback)
gc.collect()
# tasks are pruned during a send
tasks = [self.loop.create_task(signal.send(keys=[key, key2]))]
self.loop.run_until_complete(asyncio.wait(tasks))
for task in tasks:
task.result()
self.assertEqual(len(signal._by_keys), 0)
self.assertEqual(len(signal._locks_keys), 0)
def test_send_method_coro(self):
class Test:
call_count = 0
@asyncio.coroutine
def method(self, *args, **kwargs):
self.call_count += 1
instance = Test()
coro_callback = instance.method
signal = Signal(loop=self.loop)
tasks = [self.loop.create_task(signal.connect(coro_callback))]
self.loop.run_until_complete(asyncio.wait(tasks))
for task in tasks:
task.result()
tasks = [self.loop.create_task(signal.send())]
self.loop.run_until_complete(asyncio.wait(tasks))
for task in tasks:
task.result()
self.assertEqual(instance.call_count, 1)
def test_send_keys_no_args(self):
callback1 = FunctionMock()
callback2 = FunctionMock()
key1 = 'key1'
key2 = 'key2'
key3 = 'key3'
signal = Signal(loop=self.loop)
tasks = [self.loop.create_task(signal.connect(callback1, keys=[key1])),
self.loop.create_task(signal.connect(callback2, keys=[key2, key3])),
self.loop.create_task(signal.send(key=key1))]
self.loop.run_until_complete(asyncio.wait(tasks))
self.assertEqual(callback1.call_count, 1)
self.assertEqual(callback2.call_count, 0)
tasks = [self.loop.create_task(signal.send(key=key2))]
self.loop.run_until_complete(asyncio.wait(tasks))
self.assertEqual(callback1.call_count, 1)
self.assertEqual(callback2.call_count, 1)
tasks = [self.loop.create_task(signal.send(keys=[key2, key3]))]
self.loop.run_until_complete(asyncio.wait(tasks))
self.assertEqual(callback1.call_count, 1)
self.assertEqual(callback2.call_count, 2)
tasks = [self.loop.create_task(signal.send(keys=[key1, key3]))]
self.loop.run_until_complete(asyncio.wait(tasks))
self.assertEqual(callback1.call_count, 2)
self.assertEqual(callback2.call_count, 3)
def test_disconnect_all(self):
callback = FunctionMock()
key = 'some-key'
key2 = 'some-key2'
sender = object()
sender2 = object()
signal = Signal(loop=self.loop)
tasks = [self.loop.create_task(signal.connect(callback)),
self.loop.create_task(signal.connect(callback, sender=sender)),
self.loop.create_task(signal.connect(callback, senders=[sender, sender2])),
self.loop.create_task(signal.connect(callback, key=key)),
self.loop.create_task(signal.connect(callback, keys=[key, key2]))]
self.loop.run_until_complete(asyncio.wait(tasks))
# disconnect from all signals
tasks = [self.loop.create_task(signal.disconnect(callback))]
self.loop.run_until_complete(asyncio.wait(tasks))
for task in tasks:
task.result()
tasks = [self.loop.create_task(signal.send()),
self.loop.create_task(signal.send(sender=sender)),
self.loop.create_task(signal.send(senders=[sender, sender2])),
self.loop.create_task(signal.send(key=key)),
self.loop.create_task(signal.send(keys=[key, key2])), ]
self.loop.run_until_complete(asyncio.wait(tasks))
for task in tasks:
task.result()
self.assertFalse(callback.called)
def test_strongref_all_no_args_multiple(self):
fn1 = FunctionMock()
fn2 = FunctionMock()
fn3 = FunctionMock()
fn4 = FunctionMock()
signal = Signal(loop=self.loop)
tasks = [self.loop.create_task(signal.connect(fn1, weak=False)),
self.loop.create_task(signal.connect(fn2)),
self.loop.create_task(signal.connect(fn3)),
self.loop.create_task(signal.connect(fn4)),
self.loop.create_task(signal.send())]
self.loop.run_until_complete(asyncio.wait(tasks))
self.assertEqual(len(signal._all), 4)
del(fn1)
gc.collect()
# Should not be garbage collected
tasks = [self.loop.create_task(signal.send())]
self.loop.run_until_complete(asyncio.wait(tasks))
self.assertEqual(len(signal._all), 4)
def test_send_senders_no_args(self):
callback1 = FunctionMock()
callback2 = FunctionMock()
sender1 = object()
sender2 = object()
sender3 = object()
signal = Signal(loop=self.loop)
tasks = [self.loop.create_task(signal.connect(callback1, senders=[sender1])),
self.loop.create_task(signal.connect(callback2, senders=[sender2, sender3])),
self.loop.create_task(signal.send(sender=sender1))]
self.loop.run_until_complete(asyncio.wait(tasks))
self.assertEqual(callback1.call_count, 1)
self.assertEqual(callback2.call_count, 0)
tasks = [self.loop.create_task(signal.send(sender=sender2))]
self.loop.run_until_complete(asyncio.wait(tasks))
self.assertEqual(callback1.call_count, 1)
self.assertEqual(callback2.call_count, 1)
tasks = [self.loop.create_task(signal.send(senders=[sender2, sender3]))]
self.loop.run_until_complete(asyncio.wait(tasks))
self.assertEqual(callback1.call_count, 1)
self.assertEqual(callback2.call_count, 2)
tasks = [self.loop.create_task(signal.send(senders=[sender1, sender3]))]
self.loop.run_until_complete(asyncio.wait(tasks))
self.assertEqual(callback1.call_count, 2)
self.assertEqual(callback2.call_count, 3)
def test_async_await_syntax(self):
async def connect_signal(signal, callback):
await signal.connect(callback)
async def send_signal(signal):
await signal.send()
coro_callback = CoroutineMock()
signal = Signal(loop=self.loop)
tasks = [self.loop.create_task(connect_signal(signal, coro_callback))]
self.loop.run_until_complete(asyncio.wait(tasks))
# make sure no exception was raised
for task in tasks:
task.result()
# send a signal to the connected callback
tasks = [self.loop.create_task(send_signal(signal))]
self.loop.run_until_complete(asyncio.wait(tasks))
# make sure no exception was raised
for task in tasks:
task.result()
self.assertEqual(coro_callback.call_count, 1)
def update_plot(self):
try:
loop = asyncio.get_event_loop()
tasks = []
self.data
for plot in self.plots:
ax = plot['ax']
if plot['type']=='plot':
x,y = plot['args'][0], plot['args'][1]
if type(y) == str:
y = [y]
for yname,line in zip(y,ax.lines):
tasks.append(asyncio.ensure_future(self.update_line(ax, line, x, yname)))
if plot['type']=='pcolor':
x,y,z = plot['x'], plot['y'], plot['z']
tasks.append(asyncio.ensure_future(self.update_pcolor(ax, x, y, z)))
loop.run_until_complete(asyncio.wait(tasks))
display.clear_output(wait=True)
display.display(*self.figs)
time.sleep(0.1)
except KeyboardInterrupt:
loop.run_until_complete(asyncio.wait(tasks))
display.clear_output(wait=True)
display.display(*self.figs)
self._user_interrupt = True
def main():
args = argparser.parser.parse_args()
config = config_loader.load()
log = setup_logs(config, args.loglevel)
timeout = config.get('comms', 'timeout')
if args.fuzz_duration:
log.info('running fuzz test')
with Dummy(fuzz=True) as driver:
loop = asyncio.get_event_loop()
if 'TRAVIS' in os.environ:
marker = asyncio.ensure_future(log_markers())
else:
marker = asyncio.ensure_future(asyncio.sleep(0))
loop.run_until_complete(run_async_timeout(
driver, config, args.fuzz_duration, loop))
marker.cancel()
pending = asyncio.Task.all_tasks()
loop.run_until_complete(asyncio.wait(pending))
# HACK: Sleep and re-wait() is a workaround. Although wait()
# almost always returns all done/none pending, short (~1s)
# fuzz runs still often result in RuntimeError and "Task was
# destroyed but it is pending!" upon closing the loop, with
# outstanding async write()s being the offenders. Possible
# this is down to outstanding callbacks scheduled with
# call_soon() which, unlike tasks, can't be enumerated and
# cancelled/completed.
#
# A less bad workaround might be to have just one state-writer
# task which we signal through a queue and which manages its
# own write heartbeat; that way there'd be far fewer
# outstanding writes in the first place.
time.sleep(1)
# Fetch again to spot the ex nihilo task(s).
pending = asyncio.Task.all_tasks()
loop.run_until_complete(asyncio.wait(pending))
loop.close()
elif args.dummy:
log.info('running with dummy hardware')
with Dummy(fuzz=False) as driver:
run(driver, config)
elif args.emulated and not args.both:
log.info('running with emulated hardware')
from .driver.driver_emulated import Emulated
with Emulated(delay=args.delay, display_text=args.text) as driver:
run(driver, config)
elif args.emulated and args.both:
log.info('running with both emulated and real hardware on port %s'
% args.tty)
from .driver.driver_both import DriverBoth
with DriverBoth(port=args.tty, delay=args.delay,
display_text=args.text,
timeout=timeout) as driver:
run(driver, config)
else:
log.info('running with real hardware on port %s, timeout %s' %
(args.tty, timeout))
with Pi(port=args.tty, timeout=timeout) as driver:
run(driver, config)
def run(self):
'''main function to import data'''
print('处理产品现货价格')
today = datetime.now()
print('今天是', today.strftime('%Y-%m-%d'))
assets = self.config['spots']['国内现货日价格']
for asset in dict.keys(assets):
print('处理品种' + asset)
items = assets[asset]
tasks = []
for item in items:
tasks.append(asyncio.ensure_future(self.process_item('国内现货日价格', asset, item)))
try:
self.loop.run_until_complete(asyncio.wait(tasks))
except:
print('循环产生异常：正在处理产品' + asset + '国内现货价格数据')
assets = self.config['spots']['国外现货日价格']
for asset in dict.keys(assets):
print('处理品种' + asset)
items = assets[asset]
tasks = []
for item in items:
tasks.append(asyncio.ensure_future(self.process_item('国外现货日价格', asset, item)))
try:
self.loop.run_until_complete(asyncio.wait(tasks))
except:
print('循环产生异常：正在处理产品' + asset + '国外现货价格数据')
def request_greetings():
response_tasks = yield from asyncio.wait([aiohttp.get(url) for url in URLS])
text_tasks = yield from asyncio.wait(
[task.result().text() for task in response_tasks[0]]
)
texts = [task.result() for task in text_tasks[0]]
return '\n'.join(texts)
def run_under_limit(self, tasks, task_type):
"""
type could be issue/article
This function gets `ulimit -n` and tries to work under the constraint
"""
soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
count = 0
current_task = []
threshold = min(len(tasks), soft) * 0.5
for task in tasks:
current_task.append(task)
count += 1
if count >= threshold:
if task_type == 'issue':
future = asyncio.wait([self.parse_single_issue(issue_link=task, old=True)
for task in current_task])
elif task_type == 'article':
future = asyncio.wait([Article(link=link, session=self.session).save()
for link in current_task])
else:
raise TypeError("Unsupported type.")
self.loop.run_until_complete(future)
current_task = []
# Remaining tasks
if current_task:
if task_type == 'issue':
future = asyncio.wait([self.parse_single_issue(issue_link=task, old=True)
for task in current_task])
elif task_type == 'article':
future = asyncio.wait([Article(link=link, session=self.session).save()
for link in current_task])
else:
raise TypeError("Unsupported type.")
self.loop.run_until_complete(future)
def communicate_until_closed(self):
logger.info("[{}] New client.".format(self.websocket.remote_ip))
# Bring up the communication coroutines and wait for them.
# They all run infinite loops, so if any one of them completes, it
# means the client is no longer active.
communication_tasks = [asyncio.async(self._receive_to_queue()),
asyncio.async(self._send_from_queue())]
done, pending = yield from asyncio.wait(communication_tasks,
return_when=FIRST_COMPLETED)
logger.info(
"[{}] Cleaning up client...".format(self.websocket.remote_ip)
)
for task in done:
e = task.exception()
if isinstance(e, Exception):
# If any of our tasks threw an exception, re-raise it instead of
# failing silently.
raise e
# Cancel any hangers-on (viz., _send_from_queue())
for task in pending:
task.cancel()
yield from asyncio.wait(pending)
logger.info("[{}] Cleanup complete.".format(self.websocket.remote_ip))
def handle(self, *args, **options):
if options['daemonize']:
if os.fork() > 0:
sys.exit()
os.chdir('/')
os.setsid()
if os.fork() > 0:
sys.exit()
with open('/dev/null', 'r') as fh:
os.dup2(fh.fileno(), sys.stdin.fileno())
with open('/dev/null', 'a+') as fh:
os.dup2(fh.fileno(), sys.stdout.fileno())
os.dup2(fh.fileno(), sys.stderr.fileno())
loop = asyncio.get_event_loop()
from center import websocket
websocketDaemon = websocket.Main(loop)
wstask = loop.create_task(websocketDaemon.run())
from center import receiver
receiverDaemon = receiver.Main(loop)
recvtask = loop.create_task(receiverDaemon.run())
tasks = {wstask, recvtask}
loop.run_until_complete(asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED))
for t in tasks:
t.cancel()
loop.run_until_complete(asyncio.wait(tasks))
def test_ping(self):
config = kademlia.utils.load_config("config.json")
loop = asyncio.get_event_loop()
loop.set_debug(config["debug"]["asyncio"]["enabled"])
service = kademlia.Service(config, loop)
loop.run_until_complete(service.start())
loop.run_until_complete(
asyncio.wait(
[
service.tcpService.call.ping(kademlia.Remote(host="127.0.0.1", port=config["server"]["port"]))
for i in range(const.test.PING_COUNT)
]
)
)
loop.run_until_complete(
asyncio.wait([asyncio.ensure_future(self.handle_events(service))], timeout=const.test.PING_TIMEOUT)
)
loop.run_until_complete(service.stop())
self.ping_sent.sort()
self.pong_recved.sort()
self.assertEqual(len(self.ping_sent), const.test.PING_COUNT)
self.assertEqual(len(self.pong_recved), const.test.PING_COUNT)
self.assertEqual(self.ping_sent, self.pong_recved)
async def aux_func(turn_context):
LOOP.create_task(asyncio.wait([BOT.on_turn(turn_context)]))
'''
Новые задачи можно добавить в цикл,
предоставив другой объект для ожидания использования функции asyncio.wait()
Функция asyncio.wait () принимает список объектов сопрограмм
и немедленно возвращается
'''
import asyncio
async def print_number(number):
print(number)
"""
асинхронно напечатать последовательность чисел
"""
loop = asyncio.get_event_loop()
loop.run_until_complete(
asyncio.wait([print_number(number) for number in range(10)]))
loop.close()
def main_get_html(art_url):
loop = asyncio.get_event_loop() # 获取事件循环
tasks = [get_html(url) for url in art_url] # 把所有任务放到一个列表中
loop.run_until_complete(asyncio.wait(tasks)) # 激活协程
help='QUeue messages service host')
args_parser.add_argument('--rmqport',
default=5672,
help='QUeue messages service port')
args_parser.add_argument('--rmquser',
default='guest',
help='QUeue messages service user')
args_parser.add_argument('--rmqpass',
default='guest',
help='QUeue messages service password')
args_parser.add_argument('--debug',
action='store_true',
help='Debug mode turn on')
args = args_parser.parse_args()
loop = asyncio.get_event_loop()
wssrv = websocket.Server(debug=args.debug)
ws_task = websockets.serve(wssrv.handler, args.wsaddr, args.wsport)
conssrv = consumer.Server(wssrv, loop, [], args)
try:
loop.run_until_complete(asyncio.wait([ws_task, conssrv.handler()]))
except KeyboardInterrupt:
print("Keyboard interrupt signal accepted. Stopping")
finally:
loop.close()
import sys, logging
import orm, asyncio
from models import User, Blog, Comment
@asyncio.coroutine
def test(loop):
logging.error('1111111')
yield from orm.create_pool(loop=loop,
user='******',
password='******',
db='pyblog')
logging.error('2222222')
u = User(name='Test',
email='*****@*****.**',
passwd='1234567890',
image='blank')
logging.error('333333')
yield from u.save()
if __name__ == '__main__':
loop = asyncio.get_event_loop()
logging.error('beging')
loop.run_until_complete(asyncio.wait([test(loop)]))
loop.close()
if loop.is_closed():
sys.exit(0)
def get_account_list(stop_api=False, no_avatar=False):
user_path = path + "user_config/"
_list = os.listdir(user_path)
users = {}
if (no_avatar):
for account in _list:
if (account[-5:] == ".json"):
with open(user_path + account, "r") as f:
file = json.load(f)
users[str(file["steam_id"])] = file
return users
with open(path + "settings.json", "r") as f:
conf = json.load(f)
api = get_steam_web_api()
print("steamAPI status:" + str(api))
tasks = []
async def get(url, sid, type):
print("start " +
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f'))
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
req = json.loads(await response.text())
print("req " +
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f'))
if (type == "CDN"):
data = {
"persona_name": req["persona_name"],
"avatar_url": req["avatar_url"]
}
users[sid] = {**users[sid], **data}
else: # Api
req = req["response"]["players"]
for q in req:
data = {
"persona_name": q["personaname"],
"avatar_url": q["avatarfull"]
}
users[q["steamid"]] = {**users[q["steamid"]], **data}
if (api == False or stop_api == True): # use CDN
for account in _list:
if (account[-5:] == ".json"):
with open(user_path + account, "r") as f:
file = json.load(f)
users[str(file["steam_id"])] = file
url = 'https://steamcommunity.com/miniprofile/' + \
str(file["account_id"])+'/json'
tasks.append(
asyncio.ensure_future(
get(url, str(file["steam_id"]), "CDN")))
else: # webAPi
steamids = []
for account in _list:
if (account[-5:] == ".json"):
with open(user_path + account, "r") as f:
file = json.load(f)
users[str(file["steam_id"])] = file
steamids.append(str(file["steam_id"]))
def tolist():
for i in range(0, len(steamids), 100):
yield steamids[i:i + 100]
dot = ","
for user in list(tolist()):
url = 'https://api.steampowered.com/ISteamUser/GetPlayerSummaries/v0002/?key=' + \
str(conf["steam_api_key"])+'&steamids='+dot.join(user)
tasks.append(asyncio.ensure_future(get(url, "", "API")))
if (len(_list) == 0):
return {}
else:
loop.run_until_complete(asyncio.wait(tasks))
return users
logging.debug("Komplect {0} Offline".format(bus_number))
logging.debug('%s Offline'%(bus_number))
for bus in List_Checked:
#logging.debug(List_Checked)
await asyncio.wait_for(asyncio.create_task(Checked_O2(bus)),15)
#del List_Checked[bus]
#await asyncio.wait_for(asyncio.gather(*[Checked_O2(bus_number) for bus_number in List_Checked]), 15)
except Exception as e:
logging.debug("EXCEPTION : {0}".format(e))
pass
logging.debug("I Am Sleeep!!!!")
logging.debug('List_Checked {0}'.format(List_Checked))
logging.debug("End GAME")
await asyncio.sleep(10)
async def Check_Status(bus_number):
Status(bus_number)
async def Run_Check_O2():
bus_archive = Exex_1()
print ("Running Check O2")
ioloop = asyncio.get_event_loop()
tasks = [await asyncio.wait_for(ioloop.create_task(Check_O2(bus_archive)),timeout = 15)]
ioloop.run_until_complete(tasks)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait([asyncio.ensure_future(Run_Check_O2())]))
loop.close()
def main(): # 调用方
tasks = [taskIO_1(), taskIO_2()] # 把所有任务添加到task中
done,pending = yield from asyncio.wait(tasks) # 子生成器
for r in done: # done和pending都是一个任务，所以返回结果需要逐个调用result()
print("协程无序返回值：",r.result())
def envent_loop():
q = asyncio.Queue()
q.put_nowait(new_url)
loop = asyncio.get_event_loop()
tasks.append(handle_tasks(q, parse_list))
loop.run_until_complete(asyncio.wait(tasks))
def update(self, loop=None):
"""Update the safe directory.
It create the directories requires and consider filtration rules.
Copy the new files, remove the old ones.
Remove directories trees is necessary.
"""
error = list() # limit size
errors = list() # File not found
self.logger.info('Start updating')
for path_delicate, safe_path in self.safe_dirs.items():
if not path.exists(path_delicate):
break
if h.get_folder_size(
path_delicate
) > MAX_DIR_SIZE and not self.config['advanced']:
error.append(path_delicate)
continue
if safe_path['activate']:
self.logger.info(path_delicate)
safe_path_last = safe_path['LAST']
if not path.exists(safe_path_last):
self.logger.debug('Make directory: ' + safe_path_last)
h.create_dir(
safe_path_last,
self.logger) # e.g. safe_docs/my_work/my_workUPTODATE
if loop is None:
loop = asyncio.get_event_loop()
loop.run_until_complete(
asyncio.wait([
self.get_to_save(path_delicate),
self.get_saved(safe_path_last),
],
loop=loop))
dirs_to_save, dirs_maked = self.dirs_to_make, self.dirs_maked
to_save, saved = self.list_files, self.saved
# Make new folders
# dirs_to_make: new directories, not yet copying
dirs_to_make = h.missing_item(dirs_to_save, dirs_maked)
for dirname in dirs_to_make:
directory_to_create = path.join(safe_path_last, dirname)
self.logger.debug('Make directory: ' + directory_to_create)
h.create_dir(directory_to_create, self.logger)
# Copy new files
to_copy = h.missing_item(to_save, saved)
errors.extend(
self.save_files(to_copy, safe_path_last, path_delicate))
# Update existing files in safe path
to_update = h.combine_list(to_save, saved)
errors.extend(
self.update_files(to_update, safe_path_last,
path_delicate))
# Remove old files
to_del = h.missing_item(saved, to_save)
errors.extend(self.remove_files(to_del, safe_path_last))
# Delete old folders
# dirs_to_del: directories copied, deleted in delicate drectory -> to delete from safe directory
# Remove the directory tree
dirs_to_del = h.missing_item(dirs_maked, dirs_to_save)
for dirname in dirs_to_del:
self.logger.info('Remove tree: ' +
path.join(safe_path_last, dirname))
try:
rmtree(path.join(safe_path_last, dirname))
except FileNotFoundError:
pass # Directory already removed
self.logger.info('Done')
self.safe_dirs = self.get_dst_path() # !!! probleme avec activate
return error, errors
danmu_connection = connect.connect()
bili_timer = BiliTimer(loop)
console_thread = threading.Thread(target=var_console.cmdloop)
console_thread.start()
Tasks.init()
tasks = [
OnlineHeart.run(),
Silver.run(),
danmu_connection.run(),
rafflehandler.run(),
# var_console.run(),
# bili_timer.run(),
]
try:
loop.run_until_complete(asyncio.wait(tasks + list_raffle_connection_task))
except KeyboardInterrupt:
# print(sys.exc_info()[0], sys.exc_info()[1])
if ConfigLoader().dic_user['other_control']['keep-login']:
pass
else:
response = login.logout()
console_thread.join()
loop.close()
* 该函数参数也可以是 asyncio.wait(tast_arr) 函数返回值
* 表示一次性执行多个生成器
* demo :
tasks = [hello(), hello()]
loop.run_until_complete(asyncio.wait(tasks))
4,关闭 event_loop 对象 (close())
* demo
import asyncio
@asyncio.coroutine
def test(name):
print('%s 开始执行'%(name))
yield from asyncio.sleep(1)
print('%s 执行完毕' % (name))
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait([test('Kevin'),test('Litch'),test('Rocco')]))
loop.close()
Kevin 开始执行
Litch 开始执行
Rocco 开始执行
Kevin 执行完毕
Litch 执行完毕
Rocco 执行完毕
* 3.5 版本以后
* 用asyncio提供的 @asyncio.coroutine 可以把一个generator标记为coroutine类型,然后在coroutine内部用yield from调用另一个 coroutine 实现异步操作
* 为了简化并更好地标识异步IO，从Python 3.5开始引入了新的语法async和await,可以让coroutine的代码更简洁易读
* 请注意,async和await是针对coroutine的新语法,要使用新的语法,只需要做两步简单的替换
* 步骤
1,生成器不使用 @asyncio.coroutine 注解标识,而使用 async 关键字标识
def onLeave(self, details):
"""Cleanup after leaving the coordinator connection"""
if self.poll_task:
self.poll_task.cancel()
yield from asyncio.wait([self.poll_task])
super().onLeave(details)
import asyncio
import looter as lt
from pprint import pprint
domain = 'https://www.torrentkitty.tv'
keyword = 'uncensored'
limit = 500
async def crawl(url):
tree = await lt.async_fetch(url)
items = tree.css('table#archiveResult tr')[1:]
for item in items:
data = dict()
data['name'] = item.css('td.name::text').extract_first()
data['size'] = item.css('td.size::text').extract_first()
data['date'] = item.css('td.date::text').extract_first()
data['detail'] = domain + item.css(
'a[rel="information"]::attr(href)').extract_first()
data['magnet'] = item.css(
'a[rel="magnet"]::attr(href)').extract_first()
pprint(data)
if __name__ == '__main__':
tasklist = [f'{domain}/search/{keyword}/{i}' for i in range(1, limit)]
loop = asyncio.get_event_loop()
result = [crawl(task) for task in tasklist]
loop.run_until_complete(asyncio.wait(result))
cs.add(
'message', lambda args: setInterval(
lambda args: args["pushResponse"]
("Time is " + str(datetime.datetime.now())), 5, args))
async def tests():
res = await ca.ask(unitId='cServicePy',
operation='sum',
input={
'a': 5,
'b': 6
})
print("The sum is " + str(res["res"]))
ca.request('cServicePy', 'add', {"a": 555})
ca.request('cServicePy', 'add', {"a": "test"})
res = await ca.ask('cServicePy', 'list', None)
print("The list is " + str(res["res"]))
ca.persist('cServicePy', 'message', None, lambda data: print(data["res"]))
futures = [tests()]
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(futures))
# Python 3.7+
# asyncio.run()
import asyncio
async def hello():
print("Hello world!")
r = await asyncio.sleep(3)
print("Hello again!")
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait([hello()]))
url_list = []
page_content = {}
for el in page_data:
item_url, texts_arr, content_data = format_page_info(key, el)
url_list.append(item_url)
page_content[item_url] = {}
page_content[item_url]['content_data'] = content_data
page_content[item_url]['texts_arr'] = texts_arr
print('total %d/%s ' % (page, page_end))
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(
asyncio.wait([as_get_content(q, url, key) for url in url_list]))
loop.close()
while not q.empty():
item = q.get()
item_url = item[0]
item_content = item[1]
item_data = page_content.get(item_url)
new_data = item_data.get('content_data')
md5_str = md5(''.join(item_data.get('texts_arr')) +
item_content.get('content') +
item_content.get('reply'))
if collection.find_one({'content_md5': md5_str}):
page_flag = False
continue
while True:
cnt += 1
#await asyncio.sleep(1)
print("send", cnt)
t1 = datetime.utcnow()
at1 = t1 - t0
t0 = t1
print('dealy:%f' % at1.total_seconds())
A = numpy.ones((1024, 1024))
req.send(send_zipped_pickle([name, cnt, err, at1.total_seconds(), A]))
events = await poller.poll(timeout=1000 * 10)
if req in dict(events):
resp = req.recv()
print("resp:", recv_zipped_pickle(resp))
else:
err += 1
print('failed', err)
req, poller = reinitreq()
#await asyncio.sleep(1)
asyncio.get_event_loop().run_until_complete(
asyncio.wait([sender(i) for i in range(60)]))
'''
firewall-cmd --permanent --zone=public --add-port=8082/tcp
systemctl restart firewalld
'''
#running multiple coroutines
import asyncio
async def sleep_task(num):
for i in range(5):
print(f'process task {num} iter {i}')
await asyncio.sleep(1)
return num
loop = asyncio.get_event_loop()
task_list = [loop.create_task(sleep_task(i)) for i in range(2)]
loop.run_until_complete(asyncio.wait(task_list))
loop.run_until_complete(loop.create_task(sleep_task(3)))
loop.run_until_complete(asyncio.gather(
sleep_task(10),
sleep_task(20),
))
async def test_creating_updating_and_abandoning_claim_with_channel(self):
await self.account.ensure_address_gap()
address1, address2 = await self.account.receiving.get_addresses(limit=2, only_usable=True)
notifications = asyncio.create_task(asyncio.wait(
[asyncio.ensure_future(self.on_address_update(address1)),
asyncio.ensure_future(self.on_address_update(address2))]
))
await self.send_to_address_and_wait(address1, 5)
await self.send_to_address_and_wait(address2, 5, 1)
await notifications
self.assertEqual(d2l(await self.account.get_balance()), '10.0')
channel = Claim()
channel_txo = Output.pay_claim_name_pubkey_hash(
l2d('1.0'), '@bar', channel, self.account.ledger.address_to_hash160(address1)
)
channel_txo.set_channel_private_key(
await self.account.generate_channel_private_key()
)
channel_txo.script.generate()
channel_tx = await Transaction.create([], [channel_txo], [self.account], self.account)
stream = Claim()
stream.stream.source.media_type = "video/mp4"
stream_txo = Output.pay_claim_name_pubkey_hash(
l2d('1.0'), 'foo', stream, self.account.ledger.address_to_hash160(address1)
)
stream_tx = await Transaction.create([], [stream_txo], [self.account], self.account)
stream_txo.sign(channel_txo)
await stream_tx.sign([self.account])
notifications = asyncio.create_task(asyncio.wait(
[asyncio.ensure_future(self.ledger.wait(channel_tx)), asyncio.ensure_future(self.ledger.wait(stream_tx))]
))
await self.broadcast(channel_tx)
await self.broadcast(stream_tx)
await notifications
notifications = asyncio.create_task(asyncio.wait(
[asyncio.ensure_future(self.ledger.wait(channel_tx)), asyncio.ensure_future(self.ledger.wait(stream_tx))]
))
await self.generate(1)
await notifications
self.assertEqual(d2l(await self.account.get_balance()), '7.985786')
self.assertEqual(d2l(await self.account.get_balance(include_claims=True)), '9.985786')
response = await self.ledger.resolve([], ['lbry://@bar/foo'])
self.assertEqual(response['lbry://@bar/foo'].claim.claim_type, 'stream')
abandon_tx = await Transaction.create([Input.spend(stream_tx.outputs[0])], [], [self.account], self.account)
notify = asyncio.create_task(self.ledger.wait(abandon_tx))
await self.broadcast(abandon_tx)
await notify
notify = asyncio.create_task(self.ledger.wait(abandon_tx))
await self.generate(1)
await notify
response = await self.ledger.resolve([], ['lbry://@bar/foo'])
self.assertIn('error', response['lbry://@bar/foo'])
# checks for expected format in inexistent URIs
response = await self.ledger.resolve([], ['lbry://404', 'lbry://@404', 'lbry://@404/404'])
self.assertEqual('Could not find claim at "lbry://404".', response['lbry://404']['error']['text'])
self.assertEqual('Could not find channel in "lbry://@404".', response['lbry://@404']['error']['text'])
self.assertEqual('Could not find channel in "lbry://@404/404".', response['lbry://@404/404']['error']['text'])
def _loop_coroutine(
self,
on_started: Optional[Callable[["Server"], Any]] = None
) -> Generator[Any, None, None]:
try:
if self._state == State.INITIAL:
self._set_state(State.WAITING_FOR_FIRST_BROWSER)
elif self._state == State.ONE_OR_MORE_BROWSERS_CONNECTED:
pass
else:
raise RuntimeError("Bad server state at start: %s" %
self._state)
if on_started is not None:
on_started(self)
while not self._must_stop.is_set():
if self._state == State.WAITING_FOR_FIRST_BROWSER:
yield tornado.gen.convert_yielded(
asyncio.wait(
[
self._must_stop.wait(),
self._has_connection.wait()
],
return_when=asyncio.FIRST_COMPLETED,
))
elif self._state == State.ONE_OR_MORE_BROWSERS_CONNECTED:
self._need_send_data.clear()
# Shallow-clone our sessions into a list, so we can iterate
# over it and not worry about whether it's being changed
# outside this coroutine.
session_infos = list(self._session_info_by_id.values())
for session_info in session_infos:
if session_info.ws is None:
# Preheated.
continue
msg_list = session_info.session.flush_browser_queue()
for msg in msg_list:
try:
self._send_message(session_info, msg)
except tornado.websocket.WebSocketClosedError:
self._close_report_session(
session_info.session.id)
yield
yield
yield tornado.gen.sleep(0.01)
elif self._state == State.NO_BROWSERS_CONNECTED:
yield tornado.gen.convert_yielded(
asyncio.wait(
[
self._must_stop.wait(),
self._has_connection.wait()
],
return_when=asyncio.FIRST_COMPLETED,
))
else:
# Break out of the thread loop if we encounter any other state.
break
yield tornado.gen.convert_yielded(
asyncio.wait(
[self._must_stop.wait(),
self._need_send_data.wait()],
return_when=asyncio.FIRST_COMPLETED,
))
# Shut down all ReportSessions
for session_info in list(self._session_info_by_id.values()):
session_info.session.shutdown()
self._set_state(State.STOPPED)
except Exception:
# Can't just re-raise here because co-routines use Tornado
# exceptions for control flow, which appears to swallow the reraised
# exception.
traceback.print_exc()
LOGGER.info("""
Please report this bug at https://github.com/streamlit/streamlit/issues.
""")
finally:
self._on_stopped()
def wait(self):
return asyncio.wait(self._set)
def read_bytes(self, num_bytes: int) -> Awaitable[bytes]:
future = Future()
def _handle_read():
self.io_loop.remove_reader(self.socket)
data = self.socket.recv(num_bytes)
future.set_result(data)
self.io_loop.add_reader(self.socket, _handle_read)
return future
def close(self) -> None:
if self.socket:
self.socket.close()
self.socket = None
if __name__ == '__main__':
async def main():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
stream = IOStream(s)
await stream.connect(("localhost", 8000))
await stream.write(b"GET / HTTP/1.0\r\nHost: localhost\r\n\r\n")
body_data = await stream.read_bytes(1024)
print(body_data)
stream.close()
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait([main(), main()]))
def main():
search_id = "select article_id from content_article order by article_id DESC limit 1"
cur.execute(search_id)
last_id = cur.fetchone()
if last_id is None:
last_id = 0
else:
last_id = last_id[0]
print(last_id)
for open_url, article_id in zip(create_last_url([]),
range(last_id + 1, 99999)):
try:
print('开始')
print(open_url)
r3 = requests.get(open_url, headers=headers, timeout=60)
r3.encoding = 'utf-8'
thrid_html = etree.HTML(r3.text)
appid = 15
article_name = thrid_html.xpath(
'//div[@class="CBArticle"]/h1/text()')[0]
create_date = datetime.now()
create_idate = int(str(create_date.date()).replace('-', ''))
# 内容标题
article = thrid_html.xpath(
'//div[@id="NewsContentLabel"]/p//text() | //div[@id="NewsContentLabel"]//img'
)
contents, imageNum = to_img(article)
content = ''.join(contents).replace('中国娱乐网讯www.yule.com.cn', '')
# class_id = 112233
print(content)
countcollect = random.randint(1, 400)
countlike = random.randint(1, 1500)
column_id = 5
save_content_article(article_id, appid, article_name, create_date,
create_idate, open_url, content, imageNum,
countcollect, countlike)
save_content_column_article(column_id, article_id, article_name,
imageNum, appid)
# 图片处理
images_url = thrid_html.xpath(
'//div[@id="NewsContentLabel"]//img/@src')
images_url = list(set(images_url))
image_title = thrid_html.xpath(
'//div[@id="NewsContentLabel"]//img/@title')
if not image_title:
image_title = article_name
else:
image_title = image_title[0]
if images_url:
print('处理图片')
tasks = []
loop = asyncio.get_event_loop()
for image_url in images_url:
image_url = str(image_url)
path_time = str(time.time()).replace('.', '')
path_picture = 'App%d/' % (appid) + 'class%d/' % (
column_id) + '%d_' % (article_id) + path_time
picture_name = '%d_' % (article_id) + path_time
if 'http' in image_url:
pass
else:
image_url = 'http://news.yule.com.cn/yingshi' + image_url
print(picture_name)
task = asyncio.ensure_future(
save_images(image_url, picture_name))
tasks.append(task)
loop.run_until_complete(asyncio.wait(tasks))
save_content_pciture(image_title, picture_name, create_date,
create_idate, path_picture, column_id,
article_id)
save_content_picture_compress(image_title, picture_name,
create_date, create_idate,
path_picture, column_id,
article_id, countlike)
else:
print('没有image')
time.sleep(3)
except:
print("=" * 50)
print('有错误')
loop.close()
db.close()
imgs = soup.find_all('img', src=re.compile('jpg'))
title = soup.title.string.split('-')[0].strip()
urls = []
for img in imgs:
urls.append(img.get('src'))
#print(img.get('src'))
if urls:
#print(title)
yield from save(title, urls)
@asyncio.coroutine
def parse_page(url):
page = yield from aiohttp.get(url)
page = yield from page.text()
soup = bs4.BeautifulSoup(page, "html.parser")
a = soup.find_all('a', href=re.compile('reply'))
#print(url)
for link in a:
print(link)
yield from parse_url(link.get('href'))
loop = asyncio.get_event_loop()
urls = [('http://www.tuigirl8.com/home/getmore/' + str(page))
for page in range(1, 10, 1)]
loop.run_until_complete(asyncio.wait([parse_page(url) for url in urls]))
#loop.run_until_complete(parse_page('http://www.tuigirl8.com/home/getmore/1'))
#loop.run_until_complete(parse_url('http://www.tuigirl8.com/forum/view/1763'))
loop.close()
def async_update(self, *_):
"""Get the latest data from yr.no."""
import xmltodict
def try_again(err: str):
"""Retry in 15 minutes."""
_LOGGER.warning("Retrying in 15 minutes: %s", err)
self._nextrun = None
nxt = dt_util.utcnow() + timedelta(minutes=15)
if nxt.minute >= 15:
async_track_point_in_utc_time(self.hass, self.async_update,
nxt)
if self._nextrun is None or dt_util.utcnow() >= self._nextrun:
try:
websession = async_get_clientsession(self.hass)
with async_timeout.timeout(10, loop=self.hass.loop):
resp = yield from websession.get(
self._url, params=self._urlparams)
if resp.status != 200:
try_again('{} returned {}'.format(resp.url, resp.status))
return
text = yield from resp.text()
except (asyncio.TimeoutError, aiohttp.ClientError) as err:
try_again(err)
return
try:
self.data = xmltodict.parse(text)['weatherdata']
model = self.data['meta']['model']
if '@nextrun' not in model:
model = model[0]
self._nextrun = dt_util.parse_datetime(model['@nextrun'])
except (ExpatError, IndexError) as err:
try_again(err)
return
now = dt_util.utcnow()
forecast_time = now + dt_util.dt.timedelta(hours=self._forecast)
# Find the correct time entry. Since not all time entries contain all
# types of data, we cannot just select one. Instead, we order them by
# distance from the desired forecast_time, and for every device iterate
# them in order of increasing distance, taking the first time_point
# that contains the desired data.
ordered_entries = []
for time_entry in self.data['product']['time']:
valid_from = dt_util.parse_datetime(time_entry['@from'])
valid_to = dt_util.parse_datetime(time_entry['@to'])
if now >= valid_to:
# Has already passed. Never select this.
continue
average_dist = (abs((valid_to - forecast_time).total_seconds()) +
abs((valid_from - forecast_time).total_seconds()))
ordered_entries.append((average_dist, time_entry))
ordered_entries.sort(key=lambda item: item[0])
# Update all devices
tasks = []
if len(ordered_entries) > 0:
for dev in self.devices:
new_state = None
for (_, selected_time_entry) in ordered_entries:
loc_data = selected_time_entry['location']
if dev.type not in loc_data:
continue
if dev.type == 'precipitation':
new_state = loc_data[dev.type]['@value']
elif dev.type == 'symbol':
new_state = loc_data[dev.type]['@number']
elif dev.type in ('temperature', 'pressure', 'humidity',
'dewpointTemperature'):
new_state = loc_data[dev.type]['@value']
elif dev.type in ('windSpeed', 'windGust'):
new_state = loc_data[dev.type]['@mps']
elif dev.type == 'windDirection':
new_state = float(loc_data[dev.type]['@deg'])
elif dev.type in ('fog', 'cloudiness', 'lowClouds',
'mediumClouds', 'highClouds'):
new_state = loc_data[dev.type]['@percent']
break
# pylint: disable=protected-access
if new_state != dev._state:
dev._state = new_state
tasks.append(dev.async_update_ha_state())
if len(tasks) > 0:
yield from asyncio.wait(tasks, loop=self.hass.loop)
import asyncio
import time
now = lambda: time.time()
async def work(x):
print('waiting: ', x)
await asyncio.sleep(x)
return 'Done after {}s'.format(x)
def callback(future):
print('ret: ', future.result())
start = now()
tasks = []
for i in range(3):
task = asyncio.ensure_future(work(1 << i))
task.add_done_callback(callback)
tasks.append(task)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))
print('time:', now() - start)
def main():
loop = asyncio.get_event_loop()
tasks = asyncio.wait([one(), two()])
loop.run_until_complete(tasks)
# asyncio.create_task(func())
# ]
# print("main结束")
# done,pending=await asyncio.wait(task_list,timeout=None)
# print(done)
# #print("ret1:",ret1,"ret2:",ret2)
# asyncio.run(main())
#实例3:
#实例2:
import asyncio
async def func():
print(1)
await asyncio.sleep(2)
print(2)
return '返回值'
task_list = [
func(),
func(),
]
done, pending = asyncio.run(asyncio.wait(task_list))
print(done)
#print("ret1:",ret1,"ret2:",ret2)
job, err = await loop.run_in_executor(None, func, url, city, language)
errors.extend(err)
jobs.extend(job)
parsers = ((work, 'work'), (rabota, 'rabota'), (dou, 'dou'), (djinni,
'djinni'))
filters = get_filters()
url_list = get_urls(filters)
loop = asyncio.get_event_loop()
tmp_tasks = [(func, data['url_data'][key], data['language'], data['city'])
for data in url_list for func, key in parsers]
tasks = asyncio.wait([loop.create_task(task(f)) for f in tmp_tasks])
loop.run_until_complete(tasks)
loop.close()
for job in jobs:
v = Vacancy(**job)
try:
v.save()
except DatabaseError:
pass
if errors:
qs = Error.objects.filter(timestamp=datetime.date.today())
if qs.exists():
err = qs.first()