def process_queue():
if SharedData.can_process and len(SharedData.queue) > 0:
SharedData.can_process = False
current_id = next(iter(SharedData.queue))
Waiters.all_waiters.deliver_to_waiter(Waiters.WAIT_FOR_QUEUE
, {
"status": "QUEUED"
, "wallet": {"address": current_id}
})
raw_wallet = SharedData.get_raw_wallet(current_id)
if raw_wallet:
SharedData.analyze_wallet(raw_wallet)
SharedData.total_analyzed += 1
res = {
"wallet": SharedData.wallets[current_id],
"status": "DONE"
}
Waiters.all_waiters.deliver_to_waiter(Waiters.WAIT_FOR_QUEUE, res)
else:
SharedData.total_err += 1
res = {
"wallet": {"address": current_id},
"status": "ERROR"
}
Waiters.all_waiters.deliver_to_waiter(Waiters.WAIT_FOR_QUEUE, res)
SharedData.queue.remove(current_id)
if SharedData.total_analyzed % 100 == 0:
SharedData.save_wallets()
print(SharedData.total_analyzed, " analyzed, "
, SharedData.total_err, " errors, "
, SharedData.total_saved, " saved, "
, len(SharedData.queue), " in queue."
)
SharedData.can_process = True
async def recv_task(
s_receiver: asyncio.StreamReader,
q: asyncio.Queue,
e: asyncio.Event,
delimiter: bytes,
timeout=None,
):
print("[RECV][INFO] Started")
try:
while True:
try:
data = await tcp_recv(s_receiver, delimiter, timeout)
except asyncio.TimeoutError:
print('[RECV][WARN] TIMEOUT')
if e.is_set():
print(SharedData.bold(f"[RECV][INFO] Event set!"))
return
except asyncio.IncompleteReadError:
print(SharedData.red(f"[RECV][CRIT] Disconnected!"))
e.set()
return
else:
await q.put(data)
except Exception:
print(SharedData.bold("[RECV][CRIT] Stopping SEND!"))
e.set()
raise
async def send_task(
s_sender: asyncio.StreamWriter,
q: asyncio.Queue,
e: asyncio.Event,
delimiter: bytes,
timeout=None,
):
print("[SEND][INFO] Started")
try:
while True:
try:
n = await asyncio.wait_for(q.get(), timeout)
q.task_done()
except asyncio.TimeoutError:
if e.is_set():
print(SharedData.bold("[SEND][INFO] Event set!"))
return
else:
try:
await tcp_send(n, s_sender, delimiter, timeout)
except asyncio.TimeoutError:
# really just want to use logging and dump logs in other thread..
print(SharedData.red("[Send][CRIT] Connection Broken!"))
break
except Exception:
print(SharedData.bold("[SEND][CRIT] Stopping SEND!"))
e.set()
raise
def send_task(conn: socket.socket, q: Queue, stop_e: threading.Event, delimiter: bytes, timeout=None):
print("[SEND][INFO] Started")
conn.settimeout(timeout)
try:
while True:
try:
n = q.get(timeout=timeout)
q.task_done()
except Empty:
if stop_e.is_set():
print(SharedData.bold("[SEND][INFO] Event set!"))
return
else:
try:
tcp_send(n, conn, delimiter)
except socket.timeout:
# really just want to use logging and dump logs in other thread..
print(SharedData.red("[Send][CRIT] Connection Broken!"))
break
except Exception:
print(SharedData.bold("[SEND][CRIT] Stopping SEND!"))
stop_e.set()
raise
async def worker_handler(
reader: asyncio.StreamReader,
writer: asyncio.StreamWriter,
port: int,
handle_finished: asyncio.Event,
):
print(SharedData.green(f"[SS{id_:2}][INFO] --- IN HANDLER ---"))
await tcp_send(p, writer, delimiter, timeout=timeout)
print(SharedData.green(f"[SS{id_:2}][INFO] Port {port} is open."))
writer.close()
await writer.wait_closed()
handle_finished.set()
def do_turn(_game):
# _game = SharedData.game # Needed when profiling
SharedData.game = _game
start_time = _game.time_remaining()
# First turn initializations
if _game.get_turn() == 1:
SharedData.logger = Logger(DEBUG, STARTING_ON_LINE, ENDING_ON_LINE)
SharedData.ghostPirate = GhostPirate()
SharedData.attack_radius = _game.get_attack_radius()
SharedData.radius = int(math.sqrt(SharedData.attack_radius))
SharedData.locateEnemySpawnsCenters()
SharedData.islands_locations = set([isl.location for isl in _game.islands()])
SharedData.last_pirates_locs = {pirate: SharedData.deque(maxlen=3) for pirate in _game.all_my_pirates()}
# Reset turn data
SharedData.logger.last_debug_time = start_time
SharedData.pirates_dests.clear()
SharedData.enemiesInRadius_cache.clear()
SharedData.location_testing_cache.clear()
# SharedData.nearestPassableLoc.clear()
SharedData.enemy_danger_locs = {pirate: [(pirate.location[0] + r, pirate.location[1] + c)
for r in xrange(-1, 2) for c in xrange(-1, 2)
if r == 0 or c == 0] for pirate in _game.enemy_pirates()}
SharedData.my_pirates_potential_locs = {pirate: [(pirate.location[0] + r, pirate.location[1] + c)
for r in xrange(-1, 2) for c in xrange(-1, 2)
if r == 0 or c == 0] for pirate in _game.my_pirates()}
SharedData.enemy_capturing_pirates = set([pirate for pirate in _game.enemy_pirates() if _game.is_capturing(pirate)])
# Run turn
movements = Movement()
SharedData.ghostPirate.update()
if not end_game():
STRATEGY.run()
SharedData.last_enemy = set([pirate.id for pirate in filter(
lambda enemy_pirate: _game.is_passable(enemy_pirate.location), _game.enemy_pirates())])
SharedData.last_mine = set([pirate.id for pirate in filter(
lambda my_pirate: not _game.is_capturing(my_pirate), _game.my_pirates())])
movements.run()
# Time stats
turn_time = start_time - _game.time_remaining()
SharedData.max_turn_time = max(SharedData.max_turn_time, turn_time)
SharedData.logger.debug('------- TIMING: --------')
SharedData.logger.debug('This turn took {} ms / 100 ms [Max: {} ms]'.format(turn_time, SharedData.max_turn_time))
return
async def handler(recv, send):
# send config to client.
print("Sending config to client.")
await tcp_send(SharedData.load_config_raw(), send)
await handle(recv, send)
send.close()
def main():
event = threading.Event()
send_q = Queue()
recv_q = Queue()
server_thread = [
threading.Thread(target=send_thread, args=[send_q, event]),
threading.Thread(target=recv_thread, args=[recv_q, event]),
]
workers = [
threading.Thread(target=worker, args=[i, send_q, recv_q, event])
for i in range(config.WORKERS)
]
# start threads
for w in chain(server_thread, workers):
w.start()
for w in workers:
w.join()
event.set()
print(SharedData.bold("[C][info] All workers stopped."))
# send stop signal to server side RECV
print(SharedData.bold("[C][info] Sending kill signal to server RECV."))
send_q.put(config.END_MARK)
# waiting for SEND / RECV to stop
for t in server_thread:
t.join(timeout=5)
# load pickled result from INIT port
print("[C][Info] Fetching Port data from server.")
data = recv_until_eof(c_sock, config.END_MARK)
used_ports, shut_ports = json.loads(data.decode(config.ENCODING))
c_sock.close()
print("\n[Results]")
print(f"Used Ports : {used_ports}")
print(f"Closed Ports: {shut_ports}")
print(f"Excluded : {config.EXCLUDE}")
print(f"\nAll other ports from 1~{config.PORT_MAX} is open.")
def recv_thread(q: Queue, e: threading.Event):
c_sock.settimeout(TIMEOUT_FACTOR)
# making a vague assumption of timeout situation.
while True:
try:
data = c_sock.recv(65536)
except (ConnectionAbortedError, ConnectionResetError):
break
except socket.timeout:
if e.is_set():
print(SharedData.green(f"[C][RECV] Event set!"))
break
else:
if data == config.END_MARK:
print(SharedData.green(f"[C][RECV] Received eof <{data}>"))
break
q.put(read_b(data))
async def recv_task(conn: socket.socket, q: Queue, e: threading.Event, delimiter: bytes, timeout=None):
print("[RECV][INFO] Started")
conn.settimeout(timeout)
try:
while True:
try:
data = tcp_recv(conn, delimiter)
except socket.timeout:
print('[RECV][WARN] TIMEOUT')
if e.is_set():
print(SharedData.bold(f"[RECV][INFO] Event set!"))
return
else:
q.put(data)
except Exception:
print(SharedData.bold("[RECV][CRIT] Stopping SEND!"))
e.set()
raise
def send_thread(q: Queue, e: threading.Event):
while True:
try:
n = q.get(timeout=TIMEOUT_FACTOR)
except Empty:
if e.is_set():
print(SharedData.green("[C][SEND] Event set!"))
break
else:
q.task_done()
try:
c_sock.send(write_b(n))
except (ConnectionAbortedError, ConnectionResetError):
print(SharedData.red("[C][SEND] Connection reset!"))
break
except AttributeError:
print(
SharedData.red(
f"[C][SEND] Tried to send <{type(n)}> type value {n}.")
)
def worker(id_: int, send, recv, event: threading.Event):
q: Queue
send: Queue
recv: Queue
while not event.is_set():
# announce server that the worker is ready.
print(f"[CS{id_:2}][Info] Worker {id_:2} READY.")
send.put(id_)
try:
p = recv.get()
except Empty:
continue
recv.task_done()
print(f"[CS{id_:2}][Info] Worker {id_:2} received {p}.")
if p > config.PORT_MAX:
print(SharedData.cyan(f"[CS{id_:2}][Info] Stop Signal received!"))
break
print(f"[CS{id_:2}][Info] Connecting Port {p}.")
child_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
child_sock.settimeout(config.TIMEOUT)
try:
child_sock.connect((host, p))
except socket.timeout:
print(SharedData.red(f"[CS{id_:2}][Info] Port {p} Timeout."))
except OSError:
print(SharedData.red(f"[CS{id_:2}][Warn] Port {p} in use."))
else:
print(SharedData.green(f"[CS{id_:2}][Info] Port {p} is open."))
child_sock.close()
async def get_connection(handler: Callable):
ip = SharedData.get_external_ip()
while True:
try:
port = int(input("Enter primary server port >> "))
# port = 80
if port > 65535:
raise ValueError
except ValueError:
print("Wrong port value!")
continue
try:
server_co = await asyncio.start_server(handler, port=port)
except OSError:
print(SharedData.red(f"[S][CRIT] Cannot open server at port."))
raise
else:
print(f"[S][INFO] Connect client to: {ip}:{port}")
return server_co, port
async def main():
host, s_recv, s_send = await get_connection()
print(f'Config received from {host}.')
print(raw := await tcp_recv(s_recv))
config_json = json.loads(raw)
config = SimpleNamespace(**config_json)
event = asyncio.Event()
send_q = asyncio.Queue()
recv_q = asyncio.Queue()
delimiter = config.READ_UNTIL.encode(config.ENCODING)
server_task = (
asyncio.create_task(
send_task(s_send, send_q, event, delimiter, config.TIMEOUT)),
asyncio.create_task(
recv_task(s_recv, recv_q, event, delimiter, config.TIMEOUT)),
)
workers = [
asyncio.create_task(
worker(i, host, send_q, recv_q, event, delimiter, config.TIMEOUT))
for i in range(config.WORKERS)
]
# waiting for workers to complete.
for t in workers:
await t
print(SharedData.bold("[C][INFO] All workers stopped."))
if event.is_set(): # if set, then worker crashed and set the alarm!
print("task failed! waiting for server task to complete.")
else:
event.set()
for t in server_task:
try:
await t
except ConnectionResetError:
print("Connection reset!")
continue
# wait until server is closed - preventing RuntimeError.
s_send.close()
await s_send.wait_closed()
async def run_workers(
config,
works: asyncio.Queue,
send: asyncio.Queue,
recv: asyncio.Queue,
in_use: asyncio.Queue,
unreachable: asyncio.Queue,
e: asyncio.Event,
):
"""
Handle worker tasks.
"""
delimiter = config.READ_UNTIL.encode(config.ENCODING)
excl = set(config.EXCLUDE)
workers = [
asyncio.create_task(
worker(
i,
works,
send,
recv,
excl,
in_use,
unreachable,
e,
delimiter,
config.TIMEOUT,
)) for i in range(config.WORKERS)
]
for t in workers: # wait until workers are all complete
await t
print(SharedData.bold("[S][info] All workers stopped."))
import json
import threading
from itertools import chain
from queue import Queue, Empty
try:
import SharedData
except ImportError:
from sys import path
path.insert(1, "../..")
path.insert(2, "..")
import SharedData
# setup
config = SharedData.load_json_config()
c_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
TIMEOUT_FACTOR = config.SOCK_TIMEOUT
read_b, write_b = SharedData.rw_bytes(config.BYTE_SIZE, config.BYTE_ORDER,
config.END_MARK, config.ENCODING)
# Wait for connection, or a proper IP:PORT input.
while True:
host, port = input("Host [IP:Port] >> ").split(":")
# host, port = ''.split(":")
port = int(port)
try:
c_sock.connect((host, port))
except TypeError:
print(f"[C][Warn] Cannot connect to - {host}:{port}")
else:
async def main():
config = SharedData.load_json_config()
work = generate_queue(config.PORT_MAX)
delimiter = config.READ_UNTIL.encode(config.ENCODING)
timeout = config.TIMEOUT
started = asyncio.Event()
fail_trigger = asyncio.Event()
stop = asyncio.Event()
send_q = asyncio.Queue()
recv_q = asyncio.Queue()
in_use = asyncio.Queue()
failed = asyncio.Queue()
# handler for primary connection.
handle = handler_closure([send_q, recv_q], fail_trigger, started, stop,
delimiter, timeout)
async def handler(recv, send):
# send config to client.
print("Sending config to client.")
await tcp_send(SharedData.load_config_raw(), send)
await handle(recv, send)
send.close()
# start server
server_co, port_primary = await get_connection(handler)
await server_co.start_serving()
config.EXCLUDE.append(port_primary)
# wait until connection is established, and handler started.
await started.wait()
# start workers
await run_workers(config, work, send_q, recv_q, in_use, failed, stop)
fail_trigger.set()
await stop.wait()
server_co.close()
await server_co.wait_closed()
last_port = config.PORT_MAX
if not work.empty():
last_port = await work.get()
print(f"Task failed! Showing result before failed port {last_port}.")
# overriding
in_use = async_q_to_list(in_use)
failed = async_q_to_list(failed)
result = SharedData.get_port_result(in_use, failed, config.EXCLUDE,
config.PORT_MAX)
print("\n[Count Results]")
print(f"Used Ports : {result['Occupied']}")
print(f"Closed Ports: {result['Unreachable']}")
print(f"Excluded : {result['Excluded']}")
print(f"\nAll other ports from 1~{last_port} is open.")
SharedData.dump_result(result, "tcp_scan_result")
async def worker(
id_,
task_q: asyncio.Queue,
send: asyncio.Queue,
recv: asyncio.Queue,
exclude: set,
used: asyncio.Queue,
unreachable: asyncio.Queue,
event: asyncio.Event,
delimiter: bytes,
timeout=None,
):
async def worker_handler(
reader: asyncio.StreamReader,
writer: asyncio.StreamWriter,
port: int,
handle_finished: asyncio.Event,
):
print(SharedData.green(f"[SS{id_:2}][INFO] --- IN HANDLER ---"))
await tcp_send(p, writer, delimiter, timeout=timeout)
print(SharedData.green(f"[SS{id_:2}][INFO] Port {port} is open."))
writer.close()
await writer.wait_closed()
handle_finished.set()
try:
while not task_q.empty() and not event.is_set():
# receive worker announcement.
try:
worker_id = await asyncio.wait_for(recv.get(), timeout)
recv.task_done()
except asyncio.TimeoutError:
print(SharedData.red(f"[SS{id_:2}][Warn] Timeout."))
continue
print(f"[SS{id_:2}][INFO] Worker {worker_id} available.")
# get next work.
print(f"[SS{id_:2}][INFO] Getting new port.")
# if timeout getting port, either task is empty or just coroutine delayed.
try:
p: int = await asyncio.wait_for(task_q.get(), timeout)
task_q.task_done()
except asyncio.TimeoutError:
if task_q.empty():
break
else:
await recv.put(worker_id)
continue
# put back in and run again.
# check if port is in blacklist.
if p in exclude:
print(SharedData.cyan(f"[SS{id_:2}][INFO] Skipping Port {p}."))
continue
print(f"[SS{id_:2}][INFO] Sending port {p} to client.")
await send.put(p)
handle_ev = asyncio.Event()
print(f"[SS{id_:2}][INFO] Trying to serve port {p}.")
try:
# child_sock = await asyncio.wait_for(asyncio.start_server(
# lambda r, w: worker_handler(r, w, p, handle_ev), port=p), TIMEOUT_FACTOR)
child_sock = await asyncio.start_server(
lambda r, w: worker_handler(r, w, p, handle_ev), port=p)
# except asyncio.TimeoutError:
# # not sure why start_server gets timeout.
# # maybe I need to control number of task so opening server don't hang.
# print(SharedData.red(f"[SS{id_:2}][Warn] Port {p} timeout while opening."))
# await unreachable.put(p)
except AssertionError:
print(
SharedData.red(
f"[SS{id_:2}][INFO] Port {p} assertion failed!"))
await unreachable.put(p)
except OSError:
print(SharedData.red(f"[SS{id_:2}][Warn] Port {p} in use."))
await used.put(p)
else:
try:
await child_sock.start_serving()
await asyncio.wait_for(handle_ev.wait(), timeout)
except asyncio.TimeoutError:
print(
SharedData.red(f"[SS{id_:2}][Warn] Port {p} timeout."))
await unreachable.put(p)
finally:
child_sock.close()
await child_sock.wait_closed()
# Send end signal to client.
# first worker catching this signal will go offline.
print(SharedData.cyan(f"[SS{id_:2}][INFO] Done. Sending stop signal."))
await send.put("DONE"
) # causing int type-error on client side workers.
except Exception:
# trigger event to stop all threads.
print(SharedData.red(f"[SS{id_:2}][CRIT] Exception Event set!."))
event.set()
raise
if event.is_set():
print(SharedData.bold(f"[SS{id_:2}][WARN] Task Finished by event."))
else:
print(SharedData.bold(f"[SS{id_:2}][INFO] Task Finished."))
def get(self):
return json.dumps(SharedData.prepare_data())
def get(self, *args, **kwargs):
# TO-DO
self.write(json.dumps(SharedData.get_address_list()))
pass
}
Waiters.all_waiters.deliver_to_waiter(Waiters.WAIT_FOR_QUEUE, res)
else:
SharedData.total_err += 1
res = {
"wallet": {"address": current_id},
"status": "ERROR"
}
Waiters.all_waiters.deliver_to_waiter(Waiters.WAIT_FOR_QUEUE, res)
SharedData.queue.remove(current_id)
if SharedData.total_analyzed % 100 == 0:
SharedData.save_wallets()
print(SharedData.total_analyzed, " analyzed, "
, SharedData.total_err, " errors, "
, SharedData.total_saved, " saved, "
, len(SharedData.queue), " in queue."
)
SharedData.can_process = True
if __name__ == '__main__':
try:
SharedData.load_wallets()
# SharedData.start_process()
# rt = RepeatedTimer(1, process_queue)
fws = FrontWatchServer()
fws.run()
except KeyboardInterrupt:
print('Goodbye.')
async def worker(id_: int, host, send, recv, event, delimiter, timeout=None):
q: asyncio.Queue
send: asyncio.Queue
recv: asyncio.Queue
event: asyncio.Event
try:
# if one thread crashes, will trigger event and gradually stop all threads.
while not event.is_set():
# announce server that the worker is ready.
print(f"[CS{id_:2}][INFO] Worker {id_:2} READY.")
await send.put(id_)
try:
p = await asyncio.wait_for(recv.get(), timeout=timeout)
p = int(p)
recv.task_done()
except asyncio.TimeoutError:
print(
SharedData.red(
f"[CS{id_:2}][WARN] Worker {id_:2} timeout fetching from Queue."
))
continue
except ValueError:
print(
SharedData.cyan(
f"[CS{id_:2}][INFO] Stop Signal received!"))
break
print(f"[CS{id_:2}][INFO] Connecting Port {p}.")
try:
child_recv, child_send = await asyncio.wait_for(
asyncio.open_connection(host, p), timeout)
except asyncio.TimeoutError:
print(
SharedData.purple(f"[CS{id_:2}][INFO] Port {p} timeout."))
except OSError:
print(
SharedData.red(
f"[CS{id_:2}][WARN] Port {p} connection refused."))
else:
try:
print(await tcp_recv(child_recv,
delimiter,
timeout=timeout))
except asyncio.TimeoutError:
print(
SharedData.purple(
f"[CS{id_:2}][INFO] Port {p} timeout."))
except asyncio.IncompleteReadError:
print(
SharedData.red(
f"[CS{id_:2}][WARN] Port {p} disconnected!"))
else:
print(f"[CS{id_:2}][INFO] Port {p} open.")
print(
SharedData.green(
f"[CS{id_:2}][INFO] Port {p} is available."))
finally:
child_send.close()
await child_send.wait_closed()
except Exception:
# trigger event to stop all threads.
print(SharedData.red(f"[CS{id_:2}][CRIT] Exception Event set!."))
event.set()
raise
print(SharedData.bold(f"[CS{id_:2}][INFO] Task Finished."))
