def sitemap_gen(link, link_key):
if '--iocp' in sys.argv:
sys.argv.remove('--iocp')
logging.info('using iocp')
el = windows_events.ProactorEventLoop()
events.set_event_loop(el)
root_url = link
crawler(root_url, out_file=f'sitemap_{link_key}.xml')
def main():
console = logging.StreamHandler()
console.setLevel(logging.DEBUG)
console.setFormatter(logging.Formatter('%(message)s'))
log.addHandler(console)
log.setLevel(logging.DEBUG)
log.propagate = False
#logging.root.addHandler(console)
#logging.root.setLevel(logging.DEBUG)
#logging.root.propagate = False
# FIXME moved to __main__
# Should create global config
#args = ARGS.parse_args()
if ':' in args.host:
args.host, port = args.host.split(':', 1)
args.port = int(port)
if args.iocp:
from asyncio import windows_events
sys.argv.remove('--iocp')
log.info('using iocp')
el = windows_events.ProactorEventLoop()
set_event_loop(el)
if args.ssl:
here = os.path.join(os.path.dirname(__file__), 'tests')
if args.certfile:
certfile = args.certfile or os.path.join(here, 'sample.crt')
keyfile = args.keyfile or os.path.join(here, 'sample.key')
else:
certfile = os.path.join(here, 'sample.crt')
keyfile = os.path.join(here, 'sample.key')
sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
sslcontext.load_cert_chain(certfile, keyfile)
else:
sslcontext = None
loop = get_event_loop()
f = loop.create_server(lambda: HttpServer(debug=True, keep_alive=75),
args.host,
args.port,
ssl=sslcontext)
svr = loop.run_until_complete(f)
socks = svr.sockets
print('serving on', socks[0].getsockname())
try:
loop.run_forever()
except KeyboardInterrupt:
pass
def main():
args = ARGS.parse_args()
if ":" in args.host:
args.host, port = args.host.split(":", 1)
args.port = int(port)
if args.iocp:
from asyncio import windows_events
sys.argv.remove("--iocp")
logging.info("using iocp")
el = windows_events.ProactorEventLoop()
asyncio.set_event_loop(el)
if args.ssl:
here = os.path.join(os.path.dirname(__file__), "tests")
if args.certfile:
certfile = args.certfile or os.path.join(here, "sample.crt")
keyfile = args.keyfile or os.path.join(here, "sample.key")
else:
certfile = os.path.join(here, "sample.crt")
keyfile = os.path.join(here, "sample.key")
sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
sslcontext.load_cert_chain(certfile, keyfile)
else:
sslcontext = None
loop = asyncio.get_event_loop()
f = loop.create_server(
lambda: HttpRequestHandler(debug=True, keep_alive=75),
args.host,
args.port,
ssl=sslcontext,
)
svr = loop.run_until_complete(f)
socks = svr.sockets
print("serving on", socks[0].getsockname())
try:
loop.run_forever()
except KeyboardInterrupt:
pass
def function296():
var1238 = var681.parse_args()
if (':' in var1238.host):
(var1238.host, var582) = var1238.host.split(':', 1)
var1238.port = int(var582)
if var1238.iocp:
from asyncio import windows_events
sys.argv.remove('--iocp')
logging.info('using iocp')
var2853 = windows_events.ProactorEventLoop()
asyncio.set_event_loop(var2853)
if var1238.var4207:
var1682 = os.var4483.join(os.var4483.dirname(__file__), 'tests')
if var1238.var896:
var896 = (var1238.certfile
or os.var4483.join(var1682, 'sample.crt'))
var974 = (var1238.keyfile
or os.var4483.join(var1682, 'sample.key'))
else:
var896 = os.var4483.join(var1682, 'sample.crt')
var974 = os.var4483.join(var1682, 'sample.key')
var767 = var4207.SSLContext(var4207.PROTOCOL_SSLv23)
var767.load_cert_chain(var896, var974)
else:
var767 = None
var884 = asyncio.get_event_loop()
var1633 = var884.create_server(
(lambda: Class218(debug=True, keep_alive=75)),
var1238.host,
var1238.var582,
ssl=var767)
var3694 = var884.run_until_complete(var1633)
var1287 = var3694.sockets
print('serving on', var1287[0].getsockname())
try:
var884.run_forever()
except KeyboardInterrupt:
pass
print(len(self.done), 'completed tasks,', len(self.tasks),
'still pending, todo', len(self.todo))
def main():
loop = asyncio.get_event_loop()
c = Crawler(sys.argv[1], loop)
asyncio.Task(c.run())
try:
loop.add_signal_handler(signal.SIGINT, loop.stop)
except RuntimeError:
pass
loop.run_forever()
print('todo:', len(c.todo))
print('busy:', len(c.busy))
print('done:', len(c.done), '; ok:', sum(c.done.values()))
print('tasks:', len(c.tasks))
if __name__ == '__main__':
if '--iocp' in sys.argv:
from asyncio import events, windows_events
sys.argv.remove('--iocp')
logging.info('using iocp')
el = windows_events.ProactorEventLoop()
events.set_event_loop(el)
main()
def createsitemap():
if '--iocp' in sys.argv:
from asyncio import events, windows_events
sys.argv.remove('--iocp')
logging.info('using iocp')
el = windows_events.ProactorEventLoop()
import argparse
import asyncio
import aiohttp
def function1873(arg493):
var1865 = aiohttp.ClientSession()
var881 = yield from var1865.request('GET', arg493)
print(repr(var881))
var3637 = yield from var881.content.read()
print(('Downloaded: %s' % len(var3637)))
var881.close()
yield from var1865.close()
if (__name__ == '__main__'):
var627 = argparse.ArgumentParser(description='GET url example')
var627.add_argument('url', nargs=1, metavar='URL', help='URL to download')
var627.add_argument('--iocp', default=False, action='store_true', help='Use ProactorEventLoop on Windows')
var3320 = var627.parse_args()
if var3320.iocp:
from asyncio import events, windows_events
var3481 = windows_events.ProactorEventLoop()
events.set_event_loop(var3481)
var519 = asyncio.get_event_loop()
var519.run_until_complete(function1873(var3320.url[0]))
def setUp(self):
self.loop = windows_events.ProactorEventLoop()
asyncio.set_event_loop(None)
self.attribute2124[arg2097] = True
self.attribute1224.remove(arg2097)
print(len(self.attribute2124), 'completed tasks,',
len(self.attribute2337), 'still pending, todo',
len(self.attribute2317))
def function1266():
var2900 = asyncio.get_event_loop()
var759 = Class257(sys.argv[1], var2900)
asyncio.ensure_future(var759.function2883(), loop=var2900)
try:
var2900.add_signal_handler(signal.SIGINT, var2900.stop)
except RuntimeError:
pass
var2900.run_forever()
print('todo:', len(var759.attribute2317))
print('busy:', len(var759.attribute1224))
print('done:', len(var759.attribute2124), '; ok:',
sum(var759.attribute2124.values()))
print('tasks:', len(var759.attribute2337))
if (__name__ == '__main__'):
if ('--iocp' in sys.argv):
from asyncio import events, windows_events
sys.argv.remove('--iocp')
logging.info('using iocp')
var3995 = windows_events.ProactorEventLoop()
events.set_event_loop(var3995)
function1266()