def aiohttp_client_fx(db_fx: MongoClient, loop: ProactorEventLoop,
aiohttp_client: Callable[[Any], Any]) -> Any:
app = web.Application()
app.router.add_routes(urls)
app['MONGO_DB']: database.Database = db_fx
app['TOKEN_EXPIRE'] = int(environ.get('TOKEN_EXPIRE', 5))
return loop.run_until_complete(aiohttp_client(app))
def main() -> None:
"""Main program.
Parse arguments, set up event loop, run crawler, print report.
"""
args = ARGS.parse_args()
if not args.roots:
print('Use --help for command line help')
return
log = Logger(args.level)
if args.iocp:
if sys.platform == 'win32':
from asyncio import ProactorEventLoop
loop = ProactorEventLoop() # type: ignore
asyncio.set_event_loop(loop)
else:
assert False
elif args.select:
loop = asyncio.SelectorEventLoop() # type: ignore
asyncio.set_event_loop(loop)
else:
loop = asyncio.get_event_loop()
roots = {fix_url(root) for root in args.roots}
crawler = Crawler(log,
roots, exclude=args.exclude,
strict=args.strict,
max_redirect=args.max_redirect,
max_tries=args.max_tries,
max_tasks=args.max_tasks,
max_pool=args.max_pool,
)
try:
loop.run_until_complete(crawler.crawl()) # Crawler gonna crawl.
except KeyboardInterrupt:
sys.stderr.flush()
print('\nInterrupted\n')
finally:
crawler.report()
crawler.close()
loop.close()
def install_event_loop(event_loop):
if event_loop == "uvloop":
import uvloop
uvloop.install()
elif event_loop == "iocp":
from asyncio import ProactorEventLoop
set_event_loop(ProactorEventLoop())
else:
pass
def broker(ctx, in_connect, in_bind, out_connect, out_bind):
loop = LoopClass()
context = azmq.Context(loop=loop)
in_socket = context.socket(azmq.PULL)
out_socket = context.socket(azmq.PUSH)
if in_connect:
click.echo("Incoming connecting to %s." % in_connect, err=True)
in_socket.connect(in_connect)
else:
click.echo("Incoming binding on %s." % in_bind, err=True)
in_socket.bind(in_bind)
if out_connect:
click.echo("Outgoing connecting to %s." % out_connect, err=True)
out_socket.connect(out_connect)
else:
click.echo("Outgoing binding on %s." % out_bind, err=True)
out_socket.bind(out_bind)
async def run(context, in_socket, out_socket):
click.echo("Broker started.", err=True)
try:
while True:
msg = await in_socket.recv_multipart()
await out_socket.send_multipart(msg)
finally:
click.echo("Broker stopped.", err=True)
task = asyncio.ensure_future(
run(context, in_socket, out_socket),
loop=loop,
)
with allow_interruption((loop, context.close)):
loop.run_until_complete(context.wait_closed())
loop.close()
def broker(ctx, in_connect, in_bind, out_connect, out_bind):
loop = LoopClass()
context = azmq.Context(loop=loop)
in_socket = context.socket(azmq.PULL)
out_socket = context.socket(azmq.PUSH)
if in_connect:
click.echo("Incoming connecting to %s." % in_connect, err=True)
in_socket.connect(in_connect)
else:
click.echo("Incoming binding on %s." % in_bind, err=True)
in_socket.bind(in_bind)
if out_connect:
click.echo("Outgoing connecting to %s." % out_connect, err=True)
out_socket.connect(out_connect)
else:
click.echo("Outgoing binding on %s." % out_bind, err=True)
out_socket.bind(out_bind)
async def run(context, in_socket, out_socket):
click.echo("Broker started.", err=True)
try:
while True:
msg = await in_socket.recv_multipart()
await out_socket.send_multipart(msg)
finally:
click.echo("Broker stopped.", err=True)
task = asyncio.ensure_future(
run(context, in_socket, out_socket),
loop=loop,
)
with allow_interruption((loop, context.close)):
loop.run_until_complete(context.wait_closed())
loop.close()
def Console():
try:
print(Options.banner)
input("\n\tI'm not responsible for any consequence of the use of this tool, press ENTER to continue.")
Form.Validate()
if name == 'nt':
loop = ProactorEventLoop()
else:
loop = get_event_loop()
loop.run_until_complete(Attack())
loop.close()
except (KeyboardInterrupt, EOFError):
Functions.Error("Bye bye.", 1)
def main() -> None:
"""Main program.
Parse arguments, set up event loop, run crawler, print report.
"""
args = ARGS.parse_args()
if not args.roots:
print('Use --help for command line help')
return
log = Logger(args.level)
if args.iocp:
if sys.platform == 'win32':
from asyncio import ProactorEventLoop
loop = ProactorEventLoop() # type: ignore
asyncio.set_event_loop(loop)
else:
assert False
elif args.select:
loop = asyncio.SelectorEventLoop() # type: ignore
asyncio.set_event_loop(loop)
else:
loop = asyncio.get_event_loop() # type: ignore
roots = {fix_url(root) for root in args.roots}
crawler = Crawler(
log,
roots,
exclude=args.exclude,
strict=args.strict,
max_redirect=args.max_redirect,
max_tries=args.max_tries,
max_tasks=args.max_tasks,
max_pool=args.max_pool,
)
try:
loop.run_until_complete(crawler.crawl()) # Crawler gonna crawl.
except KeyboardInterrupt:
sys.stderr.flush()
print('\nInterrupted\n')
finally:
crawler.report()
crawler.close()
loop.close()
""" Application main window close action """
for queue in all_tasks():
queue.cancel()
class Canvas(FigureCanvas):
""" """
def __init__(self, parent):
fig = Figure()
fig.patch.set_facecolor(parent.palette().color(QPalette.Background).name())
FigureCanvas.__init__(self, fig)
self.setParent(parent)
self.ax = self.figure.add_subplot(111, projection='polar')
if __name__ == '__main__':
_application = QApplication(argv)
_loop = ProactorEventLoop()
set_event_loop(_loop)
_ui = ApplicationUI()
_ui.show()
try:
_loop.run_until_complete(_processing(_application))
except CancelledError:
exit(0)
def get_event_loop(self):
return ProactorEventLoop()
async def __main(self, loop: asyncio.ProactorEventLoop):
"""This instance handles the websocket connections."""
async def event_loop():
async def execute_listener(listener: str, *args):
listener = self.__listeners.get(listener)
if listener:
asyncio.ensure_future(listener[0](
*args) if listener[1] else asyncio.create_task(
listener[0](self, *args)))
info("Dogehouse: Starting event listener loop")
while self.__active:
res = loads(await self.__socket.recv())
op = res if isinstance(res, str) else res.get("op")
if op == "auth-good":
info("Dogehouse: Received client ready")
self.user = User.from_dict(res["d"]["user"])
await execute_listener("on_ready")
elif op == "new-tokens":
info("Dogehouse: Received new authorization tokens")
self.__token = res["d"]["accessToken"]
self.__refresh_token = res["d"]["refreshToken"]
elif op == "fetch_done":
fetch = self.__fetches.get(res.get("fetchId"), False)
if fetch:
del self.__fetches[res.get("fetchId")]
if fetch == "get_top_public_rooms":
info("Dogehouse: Received new rooms")
self.rooms = list(
map(Room.from_dict, res["d"]["rooms"]))
elif fetch == "create_room":
info("Dogehouse: Created new room")
self.room = Room.from_dict(res["d"]["room"])
elif op == "you-joined-as-speaker":
await execute_listener("on_room_join", True)
elif op == "join_room_done":
self.room = Room.from_dict(res["d"]["room"])
await execute_listener("on_room_join", False)
elif op == "new_user_join_room":
await execute_listener("on_user_join",
User.from_dict(res["d"]["user"]))
elif op == "user_left_room":
await self.get_top_public_rooms()
await execute_listener("on_user_leave", res["d"]["userId"])
elif op == "new_chat_msg":
msg = Message.from_dict(res["d"]["msg"])
await execute_listener("on_message", msg)
async def heartbeat():
debug("Dogehouse: Starting heartbeat")
while self.__active:
await self.__socket.send("ping")
await asyncio.sleep(heartbeatInterval)
async def get_top_rooms_loop():
debug("Dogehouse: Starting to get all rooms")
while self.__active and not self.room:
await self.get_top_public_rooms()
await asyncio.sleep(topPublicRoomsInterval)
try:
info("Dogehouse: Connecting with Dogehouse websocket")
async with websockets.connect(apiUrl) as ws:
info(
"Dogehouse: Websocket connection established successfully")
self.__active = True
self.__socket = ws
info("Dogehouse: Attemting to authenticate")
await self.__send(
'auth', {
"accessToken": self.__token,
"refreshToken": self.__refresh_token,
"reconnectToVoice": self.__reconnect_voice,
"muted": self.__muted,
"currentRoomId": self.room,
"platform": "dogehouse.py"
})
info("Dogehouse: Successfully authenticated")
event_loop_task = loop.create_task(event_loop())
get_top_rooms_task = loop.create_task(get_top_rooms_loop())
await heartbeat()
await event_loop_task()
await get_top_rooms_task()
except ConnectionClosedOK:
info("Dogehouse: Websocket connection closed peacefully")
self.__active = False
except ConnectionClosedError as e:
if (e.code == 4004):
raise InvalidAccessToken()
for graph in config.graph_types:
for idx, signal in enumerate(
getattr(config, graph + '_output')):
signal_curve = getattr(self, signal + '_curve')
signal_data = getattr(self, signal + '_data')
signal_color = getattr(config, graph + '_color')
signal_data.append(int(''.join(clear_data[signal]),
16))
signal_curve.setData(signal_data,
pen=mkPen(
width=1.5,
color=signal_color[idx]))
await sleep(1)
if __name__ == '__main__':
app = QApplication(argv)
# Preparing the main loop
loop = ProactorEventLoop()
set_event_loop(loop)
# Preparing and displaying the GUI
gui = ProgramUI(loop)
gui.show()
try:
loop.run_until_complete(async_process_subprocess(app))
except CancelledError:
pass
async def proactor_wrap(loop_: asyncio.ProactorEventLoop,
fut: asyncio.coroutines):
await fut
loop_.stop()
def client(ctx, in_connect, in_bind, out_connect, out_bind, count, size):
in_loop = LoopClass()
in_context = azmq.Context(loop=in_loop)
in_socket = in_context.socket(azmq.PULL)
if in_connect:
click.echo("Incoming connecting to %s." % in_connect, err=True)
in_socket.connect(in_connect)
else:
click.echo("Incoming binding on %s." % in_bind, err=True)
in_socket.bind(in_bind)
out_loop = LoopClass()
out_context = azmq.Context(loop=out_loop)
out_socket = out_context.socket(azmq.PUSH)
out_socket.max_outbox_size = 1
if out_connect:
click.echo("Outgoing connecting to %s." % out_connect, err=True)
out_socket.connect(out_connect)
else:
click.echo("Outgoing binding on %s." % out_bind, err=True)
out_socket.bind(out_bind)
in_done = asyncio.Event(loop=out_loop)
out_done = asyncio.Event(loop=in_loop)
data_lock = threading.Lock()
data_results = {}
async def run_sender(context, socket, msg, count):
# The list of sending times.
send_times = [None] * count
async with context:
# Leave some time for the connection to be established.
click.echo("Sender started.", err=True)
try:
for i in range(count):
send_times[i] = perf_counter()
await socket.send_multipart(msg)
finally:
in_loop.call_soon_threadsafe(out_done.set)
click.echo("Sender stopping...", err=True)
await in_done.wait()
click.echo("Sender stopped.", err=True)
with data_lock:
data_results.update({
'send_times': send_times,
})
async def run_receiver(context, socket, msg, count):
click.echo("Receiver started.", err=True)
# The list of receiving times.
receive_times = [None] * count
async with context:
try:
for i in range(count):
await socket.recv_multipart()
receive_times[i] = perf_counter()
finally:
out_loop.call_soon_threadsafe(in_done.set)
click.echo("Receiver stopping...", err=True)
await out_done.wait()
click.echo("Receiver stopped.", err=True)
with data_lock:
data_results.update({
'receive_times': receive_times,
})
msg = [b'x' * size]
out_task = asyncio.ensure_future(
run_sender(out_context, out_socket, msg, count),
loop=out_loop,
)
in_task = asyncio.ensure_future(
run_receiver(in_context, in_socket, msg, count),
loop=in_loop,
)
in_thread = threading.Thread(
target=in_loop.run_until_complete,
args=(in_context.wait_closed(), ),
)
out_thread = threading.Thread(
target=out_loop.run_until_complete,
args=(out_context.wait_closed(), ),
)
in_thread.start()
out_thread.start()
with allow_interruption(
(in_loop, in_context.close),
(out_loop, out_context.close),
):
out_thread.join()
in_thread.join()
in_loop.run_until_complete(
asyncio.gather(in_task, return_exceptions=True, loop=in_loop), )
out_loop.run_until_complete(
asyncio.gather(out_task, return_exceptions=True, loop=out_loop), )
in_loop.close()
out_loop.close()
click.echo("Computing results...", err=True)
keys = list(data_results.keys())
values = zip(*data_results.values())
for index, value in enumerate(values):
raw_data = dict(zip(keys, value))
data = {
'sent': raw_data['send_times'],
'received': raw_data['receive_times'],
'latency': raw_data['receive_times'] - raw_data['send_times'],
'size': size,
}
sys.stdout.write(json.dumps(index))
sys.stdout.write('\t')
sys.stdout.write(json.dumps(data))
sys.stdout.write('\n')
click.echo("Done computing results.", err=True)
async def __main(self, loop: asyncio.ProactorEventLoop):
"""This instance handles the websocket connections."""
async def event_loop():
async def execute_listener(listener: str, *args):
listener = self.__listeners.get(listener.lower())
if listener:
asyncio.ensure_future(listener[0](
*args) if listener[1] else listener[0](self, *args))
async def execute_command(command_name: str, ctx: Message, *args):
command = self.__commands.get(command_name.lower())
if command:
arguments = []
params = {}
parameters = list(signature(command[0]).parameters.items())
if not command[1]:
arguments.append(self)
parameters.pop(0)
if parameters:
arguments.append(ctx)
parameters.pop(0)
for idx, (key, param) in enumerate(parameters):
value = args[idx]
if param.kind == param.KEYWORD_ONLY:
value = " ".join(args[idx::])
params[key] = value
try:
asyncio.ensure_future(command[0](*arguments, **params))
except TypeError:
raise NotEnoughArguments(
f"Not enough arguments were provided in command `{command_name}`."
)
else:
raise CommandNotFound(
f"The requested command `{command_name}` does not exist."
)
info("Dogehouse: Starting event listener loop")
while self.__active:
res = loads(await self.__socket.recv())
op = res if isinstance(res, str) else res.get("op")
if op == "auth-good":
info("Dogehouse: Received client ready")
self.user = User.from_dict(res["d"]["user"])
await execute_listener("on_ready")
elif op == "new-tokens":
info("Dogehouse: Received new authorization tokens")
self.__token = res["d"]["accessToken"]
self.__refresh_token = res["d"]["refreshToken"]
elif op == "fetch_done":
fetch = self.__fetches.get(res.get("fetchId"), False)
if fetch:
del self.__fetches[res.get("fetchId")]
if fetch == "get_top_public_rooms":
info("Dogehouse: Received new rooms")
self.rooms = list(
map(Room.from_dict, res["d"]["rooms"]))
elif fetch == "create_room":
info("Dogehouse: Created new room")
self.room = Room.from_dict(res["d"]["room"])
elif op == "you-joined-as-speaker":
await execute_listener("on_room_join", True)
elif op == "join_room_done":
self.room = Room.from_dict(res["d"]["room"])
await execute_listener("on_room_join", False)
elif op == "new_user_join_room":
await execute_listener("on_user_join",
User.from_dict(res["d"]["user"]))
elif op == "user_left_room":
await self.get_top_public_rooms()
await execute_listener("on_user_leave", res["d"]["userId"])
elif op == "new_chat_msg":
msg = Message.from_dict(res["d"]["msg"])
await execute_listener("on_message", msg)
try:
if msg.content.startswith(self.prefix) and len(
msg.content) > len(self.prefix) + 1:
splitted = msg.content[len(self.prefix)::].split(
" ")
await execute_command(splitted[0], msg,
*splitted[1::])
except Exception as e:
await execute_listener("on_error", e)
async def heartbeat():
debug("Dogehouse: Starting heartbeat")
while self.__active:
await self.__socket.send("ping")
await asyncio.sleep(heartbeatInterval)
async def get_top_rooms_loop():
debug("Dogehouse: Starting to get all rooms")
while self.__active and not self.room:
await self.get_top_public_rooms()
await asyncio.sleep(topPublicRoomsInterval)
try:
info("Dogehouse: Connecting with Dogehouse websocket")
async with websockets.connect(apiUrl) as ws:
info(
"Dogehouse: Websocket connection established successfully")
self.__active = True
self.__socket = ws
info("Dogehouse: Attemting to authenticate")
await self.__send(
'auth', {
"accessToken": self.__token,
"refreshToken": self.__refresh_token,
"reconnectToVoice": self.__reconnect_voice,
"muted": self.__muted,
"currentRoomId": self.room,
"platform": "dogehouse.py"
})
info("Dogehouse: Successfully authenticated")
event_loop_task = loop.create_task(event_loop())
get_top_rooms_task = loop.create_task(get_top_rooms_loop())
await heartbeat()
await event_loop_task()
await get_top_rooms_task()
except ConnectionClosedOK:
info("Dogehouse: Websocket connection closed peacefully")
self.__active = False
except ConnectionClosedError as e:
if (e.code == 4004):
raise InvalidAccessToken()
async def main(slave_path: str) -> None:
tasks = [run_controller(slave_path) for idx in range(5)]
await gather(*tasks)
if __name__ == "__main__":
slave_path = get_params().slavePath
try:
if slave_path and not isfile(slave_path):
raise FileNotFoundError(
"ERROR: '--slavePath' parameter is not a file."
f" Bad path: {slave_path}")
except FileNotFoundError as error:
print(error)
exit(2)
# Need to change eventloop for windows to run asyncio subprocesses
if platform == "win32":
loop = ProactorEventLoop()
set_event_loop(loop)
loop.run_until_complete(main(slave_path))
loop.close()
else:
run(main(slave_path))
config_file = pathlib.Path(root / "config.ini")
sql_session = pathlib.Path(root / "userbot.session")
ROOT_LOGGER = logging.getLogger()
LOGGER = logging.getLogger(__name__)
streamHandler = logging.StreamHandler()
streamHandler.setFormatter(CustomFormatter(datefmt="%X"))
loggingHandler = CustomMemoryHandler(600, target=streamHandler)
ROOT_LOGGER.addHandler(loggingHandler)
logging.captureWarnings(True)
if sys.platform.startswith("win"):
from asyncio import ProactorEventLoop
loop = ProactorEventLoop()
os.system("color")
os.system("cls")
else:
os.system("clear")
if platform.python_version_tuple() < ("3", "7", "3"):
print("Please run this script with Python 3.7.3 or above."
"\nExiting the script.")
sys.exit(1)
if config_file.exists():
config.read(config_file)
resolve_env(config)
try:
def client(ctx, in_connect, in_bind, out_connect, out_bind, count, size):
in_loop = LoopClass()
in_context = azmq.Context(loop=in_loop)
in_socket = in_context.socket(azmq.PULL)
if in_connect:
click.echo("Incoming connecting to %s." % in_connect, err=True)
in_socket.connect(in_connect)
else:
click.echo("Incoming binding on %s." % in_bind, err=True)
in_socket.bind(in_bind)
out_loop = LoopClass()
out_context = azmq.Context(loop=out_loop)
out_socket = out_context.socket(azmq.PUSH)
out_socket.max_outbox_size = 1
if out_connect:
click.echo("Outgoing connecting to %s." % out_connect, err=True)
out_socket.connect(out_connect)
else:
click.echo("Outgoing binding on %s." % out_bind, err=True)
out_socket.bind(out_bind)
in_done = asyncio.Event(loop=out_loop)
out_done = asyncio.Event(loop=in_loop)
data_lock = threading.Lock()
data_results = {}
async def run_sender(context, socket, msg, count):
# The list of sending times.
send_times = [None] * count
async with context:
# Leave some time for the connection to be established.
click.echo("Sender started.", err=True)
try:
for i in range(count):
send_times[i] = perf_counter()
await socket.send_multipart(msg)
finally:
in_loop.call_soon_threadsafe(out_done.set)
click.echo("Sender stopping...", err=True)
await in_done.wait()
click.echo("Sender stopped.", err=True)
with data_lock:
data_results.update({
'send_times': send_times,
})
async def run_receiver(context, socket, msg, count):
click.echo("Receiver started.", err=True)
# The list of receiving times.
receive_times = [None] * count
async with context:
try:
for i in range(count):
await socket.recv_multipart()
receive_times[i] = perf_counter()
finally:
out_loop.call_soon_threadsafe(in_done.set)
click.echo("Receiver stopping...", err=True)
await out_done.wait()
click.echo("Receiver stopped.", err=True)
with data_lock:
data_results.update({
'receive_times': receive_times,
})
msg = [b'x' * size]
out_task = asyncio.ensure_future(
run_sender(out_context, out_socket, msg, count),
loop=out_loop,
)
in_task = asyncio.ensure_future(
run_receiver(in_context, in_socket, msg, count),
loop=in_loop,
)
in_thread = threading.Thread(
target=in_loop.run_until_complete,
args=(in_context.wait_closed(),),
)
out_thread = threading.Thread(
target=out_loop.run_until_complete,
args=(out_context.wait_closed(),),
)
in_thread.start()
out_thread.start()
with allow_interruption(
(in_loop, in_context.close),
(out_loop, out_context.close),
):
out_thread.join()
in_thread.join()
in_loop.run_until_complete(
asyncio.gather(in_task, return_exceptions=True, loop=in_loop),
)
out_loop.run_until_complete(
asyncio.gather(out_task, return_exceptions=True, loop=out_loop),
)
in_loop.close()
out_loop.close()
click.echo("Computing results...", err=True)
keys = list(data_results.keys())
values = zip(*data_results.values())
for index, value in enumerate(values):
raw_data = dict(zip(keys, value))
data = {
'sent': raw_data['send_times'],
'received': raw_data['receive_times'],
'latency': raw_data['receive_times'] - raw_data['send_times'],
'size': size,
}
sys.stdout.write(json.dumps(index))
sys.stdout.write('\t')
sys.stdout.write(json.dumps(data))
sys.stdout.write('\n')
click.echo("Done computing results.", err=True)