def main():
global args
args = ARGS.parse_args()
if args.iocp:
from trollius.windows_events import ProactorEventLoop
loop = ProactorEventLoop()
set_event_loop(loop)
else:
loop = get_event_loop()
try:
loop.run_until_complete(start(loop, args.host, args.port))
finally:
loop.close()
def main():
if '--iocp' in sys.argv:
from trollius.windows_events import ProactorEventLoop
loop = ProactorEventLoop()
set_event_loop(loop)
else:
loop = get_event_loop()
try:
body = loop.run_until_complete(fetch(sys.argv[1], '-v' in sys.argv))
finally:
loop.close()
if hasattr(sys.stdout, 'buffer'):
sys.stdout.buffer.write(body)
else:
# Python 2
sys.stdout.write(body)
def main():
if '--iocp' in sys.argv:
from trollius.windows_events import ProactorEventLoop
loop = ProactorEventLoop()
set_event_loop(loop)
else:
loop = get_event_loop()
try:
body = loop.run_until_complete(fetch(sys.argv[1], '-v' in sys.argv))
finally:
loop.close()
if hasattr(sys.stdout, 'buffer'):
sys.stdout.buffer.write(body)
else:
# Python 2
sys.stdout.write(body)
def main():
asyncio.set_event_loop(None)
if args.iocp:
from trollius.windows_events import ProactorEventLoop
loop = ProactorEventLoop()
else:
loop = asyncio.new_event_loop()
sslctx = None
if args.tls:
import ssl
# TODO: take cert/key from args as well.
here = os.path.join(os.path.dirname(__file__), '..', 'tests')
sslctx = asyncio.SSLContext(ssl.PROTOCOL_SSLv23)
sslctx.options |= ssl.OP_NO_SSLv2
sslctx.load_cert_chain(certfile=os.path.join(here, 'ssl_cert.pem'),
keyfile=os.path.join(here, 'ssl_key.pem'))
cache = Cache(loop)
task = asyncio.streams.start_server(cache.handle_client,
args.host,
args.port,
ssl=sslctx,
loop=loop)
svr = loop.run_until_complete(task)
for sock in svr.sockets:
logging.info('socket %s', sock.getsockname())
try:
loop.run_forever()
finally:
loop.close()
def main():
asyncio.set_event_loop(None)
if args.iocp:
from trollius.windows_events import ProactorEventLoop
loop = ProactorEventLoop()
else:
loop = asyncio.new_event_loop()
sslctx = None
if args.tls:
sslctx = test_utils.dummy_ssl_context()
cache = CacheClient(args.host, args.port, sslctx=sslctx, loop=loop)
try:
loop.run_until_complete(
asyncio.gather(
*[testing(i, cache, loop) for i in range(args.ntasks)],
loop=loop))
finally:
loop.close()
def main():
"""Main program.
Parse arguments, set up event loop, run crawler, print report.
"""
args = ARGS.parse_args()
if not args.roots:
print('Use --help for command line help')
return
log = Logger(args.level)
if args.iocp:
from trollius.windows_events import ProactorEventLoop
loop = ProactorEventLoop()
asyncio.set_event_loop(loop)
elif args.select:
loop = asyncio.SelectorEventLoop()
asyncio.set_event_loop(loop)
else:
loop = asyncio.get_event_loop()
roots = {fix_url(root) for root in args.roots}
crawler = Crawler(log,
roots, exclude=args.exclude,
strict=args.strict,
max_redirect=args.max_redirect,
max_tries=args.max_tries,
max_tasks=args.max_tasks,
max_pool=args.max_pool,
)
try:
loop.run_until_complete(crawler.crawl()) # Crawler gonna crawl.
except KeyboardInterrupt:
sys.stderr.flush()
print('\nInterrupted\n')
finally:
crawler.report()
crawler.close()
loop.close()
def main():
asyncio.set_event_loop(None)
if args.iocp:
from trollius.windows_events import ProactorEventLoop
loop = ProactorEventLoop()
else:
loop = asyncio.new_event_loop()
sslctx = None
if args.tls:
import ssl
# TODO: take cert/key from args as well.
here = os.path.join(os.path.dirname(__file__), '..', 'tests')
sslctx = asyncio.SSLContext(ssl.PROTOCOL_SSLv23)
sslctx.options |= ssl.OP_NO_SSLv2
sslctx.load_cert_chain(
certfile=os.path.join(here, 'ssl_cert.pem'),
keyfile=os.path.join(here, 'ssl_key.pem'))
cache = Cache(loop)
task = asyncio.streams.start_server(cache.handle_client,
args.host, args.port,
ssl=sslctx, loop=loop)
svr = loop.run_until_complete(task)
for sock in svr.sockets:
logging.info('socket %s', sock.getsockname())
try:
loop.run_forever()
finally:
loop.close()
def main():
global args
args = ARGS.parse_args()
if args.iocp:
from trollius.windows_events import ProactorEventLoop
loop = ProactorEventLoop()
set_event_loop(loop)
else:
loop = get_event_loop()
try:
loop.run_until_complete(start(loop, args.host, args.port))
finally:
loop.close()
def main():
"""Main program.
Parse arguments, set up event loop, run crawler, print report.
"""
args = ARGS.parse_args()
if not args.roots:
print('Use --help for command line help')
return
log = Logger(args.level)
if args.iocp:
from trollius.windows_events import ProactorEventLoop
loop = ProactorEventLoop()
asyncio.set_event_loop(loop)
elif args.select:
loop = asyncio.SelectorEventLoop()
asyncio.set_event_loop(loop)
else:
loop = asyncio.get_event_loop()
roots = {fix_url(root) for root in args.roots}
crawler = Crawler(
log,
roots,
exclude=args.exclude,
strict=args.strict,
max_redirect=args.max_redirect,
max_tries=args.max_tries,
max_tasks=args.max_tasks,
max_pool=args.max_pool,
)
try:
loop.run_until_complete(crawler.crawl()) # Crawler gonna crawl.
except KeyboardInterrupt:
sys.stderr.flush()
print('\nInterrupted\n')
finally:
crawler.report()
crawler.close()
loop.close()
# get and print lines from stdout, stderr
timeout = None
while registered:
done, pending = yield From(asyncio.wait(
registered, timeout=timeout,
return_when=asyncio.FIRST_COMPLETED))
if not done:
break
for f in done:
stream = registered.pop(f)
res = f.result()
print(name[stream], res.decode('ascii').rstrip())
if res != b'':
registered[asyncio.Task(stream.readline())] = stream
timeout = 0.0
stdout_transport.close()
stderr_transport.close()
if __name__ == '__main__':
if sys.platform == 'win32':
loop = ProactorEventLoop()
asyncio.set_event_loop(loop)
else:
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(main(loop))
finally:
loop.close()
timeout = None
while registered:
done, pending = yield From(
asyncio.wait(registered,
timeout=timeout,
return_when=asyncio.FIRST_COMPLETED))
if not done:
break
for f in done:
stream = registered.pop(f)
res = f.result()
print(name[stream], res.decode('ascii').rstrip())
if res != b'':
registered[asyncio.Task(stream.readline())] = stream
timeout = 0.0
stdout_transport.close()
stderr_transport.close()
if __name__ == '__main__':
if sys.platform == 'win32':
loop = ProactorEventLoop()
asyncio.set_event_loop(loop)
else:
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(main(loop))
finally:
loop.close()
