def write(self, txt):
if (threading.currentThread() in Pyto.ignoredThreads):
return
oldStdout.write(txt)
Pyto.print(txt, end="")
def read(text):
try:
console
except NameError:
import console
console.print(text, end="")
def _print(start, end, size):
nonlocal total
total += size
print(
f'\rProgress: {con.pretty_size(total)} / {con.pretty_size(self.file_size)},'
f'{con.pretty_size(size)}ps, Chunk[{start}/{end}]',
sep=' ',
end='')
def on_create(self) -> None:
con.clear()
con.box_print("Project Malcolm")
print("Local IP Address:", network.get_local_ip())
print("Public IP Address:", network.get_public_ip())
print("Server IP Address:", self.address)
print()
print("Press any key to continue")
con.getch()
async def get_checksum(self) -> None:
counter: int = 0
def _print(start, end, size):
nonlocal counter
counter += 1
print(f'\rProgress: {counter}/{len(self.ports)}', end='')
self.generate_connections()
self.ports, self.checks = map(
list,
zip(*[(
port,
str(check)) for port, check in zip(self.ports, [
__check for __check in flatten(await asyncio.gather(*(
self._async_get(soc, Request.CHECKSUM, progress=_print)
for soc in self.conns)))
]) if check is not None]))
print() # Required due to \r in _print
def display_info():
global procs, queue, status
while True:
con.clear()
try:
for index, (process_id, process, queue) in enumerate(procs):
try:
status[index] = queue.get_nowait()
except Empty:
pass
print('\r', status[index])
except KeyboardInterrupt as e:
for process, queue in procs:
process.terminate()
process.join()
break
except Exception as e:
con.error(e)
else:
time.sleep(args.interval)
def verify(args: argparse.Namespace):
con.pretty_printing = args.colors
try:
print("Validating IP Address ... ", end="")
ip = args.address.split(".")
# Check if IP has 4 decimal point
assert len(ip) == 4
# Check if any of them is empty
for ad in ip:
assert len(ad) != 0
assert network.ping(args.address)
con.success("OK")
# Verify if the path given is valid directory
print("Validating directory ... ", end="")
assert pathlib.Path(args.output).is_dir()
con.success("OK")
# Check if resume flag is enabled
if args.resume:
# Check if resume.json exists
print("Validating resume data ... ", end="")
assert pathlib.Path('resume.pyb').exists()
con.success("OK")
return args
except AssertionError:
con.error("FAILED")
quit(1)
def verify_checksum(self) -> None:
assert self.checks is not None
_checks = sorted(self.checks)
check_count = []
# Group the same checksum ad make tuple with there count
for _, g_iter in groupby(_checks):
tmp = list(g_iter)
check_count.append((len(tmp), tmp[0]))
# Find the max number of same checksum and return the checksum
check_selected = max(check_count, key=itemgetter(0))[1]
# Filter the ports whose checksum doesn't match
self.ports, self.checks = map(
list,
zip(*[(port, check) for port, check in zip(self.ports, _checks)
if check == check_selected]))
# Check if there is resume data
if self.resume_data is not None:
# Match the checksum
for check in self.checks:
if check != self.resume_data.checks:
print(
'The resume data checksum does not match with server sent checksum'
)
print(f'{check} != {self.resume_data.checks}')
print('Terminating ...')
quit(1)
def generate_connections(self,
*,
sockets: Optional[Sequence[int]] = None) -> None:
# Close all the existing sockets
_conns: Optional[MutableSequence[Optional[socket.socket]]] = None
try:
for soc in self.conns:
soc.close()
# Check if only selected sockets needs to be recreated
_conns = None if self.conns is None else [
optional(c) for c in self.conns
]
except AttributeError:
pass
if _conns is None or sockets is None:
# Recreate all the sockets
_conns = [
network.create_connection(self.address, int(port))
for port in self.ports
]
else:
# Recreate the sockets given in sockets sequence parameter
for sock in sockets:
_conns[sock] = network.create_connection(
self.address, int(self.ports[sock]))
# Remove all the ports that have None as a connection
self.ports = [
port for port, conn in zip(self.ports, _conns) if conn is not None
]
# Remove all the None connections
self.conns = [conn for conn in _conns if conn is not None]
# Check if there are any working ports
# If a port is in list, it means it was working previously
if empty(self.ports):
# If no port is available, then quit
print('All connections refused ...')
# Call resume method to make sure data is stored for resume
self.dump_resume()
quit(1)
def dump_resume(self):
# Check if the data exists
save_resume_data: bool = False
try:
# Destructive unpacking of the tuple
# Discarding start and end since they are not required
for (_, _), data in self.data_unfinished:
# Check if there is any element is None
# Since there is no need to create resume file otherwise
if data is not None:
save_resume_data = True
except (NameError, ValueError, AttributeError):
# Exists if the variable doesn't exists or data is incompatible
pass
# If there is data
if save_resume_data:
print('Saving resume data ...')
with open('resume.pyb', 'wb') as f:
# Dump the data using serialization hook
dump(ClientInfo(first(self.checks), self.data_unfinished), f)
else:
print('No data was fetched ...')
def verify(args: argparse.Namespace):
con.pretty_printing = args.colors
try:
# Verify number of ports equal the number of servers
print("Validating server count ... ", end="")
assert args.number == len(args.ports)
con.success("OK")
print("Validating server ports ... ")
for port in args.ports:
print("\t{} ... ".format(port), end="")
assert network.check_sock(network.get_local_ip(), port)
con.success("OK")
# Verify a valid file was passed
print("Validating file ... ", end="")
assert pathlib.Path(args.file).is_file()
con.success("OK")
return args
except AssertionError:
con.error("FAILED")
quit(1)
def print_error(err_str):
con.print('%s: 错误: %s' % ('myaip', err_str), con.LRED)
def write(self, txt):
oldStdout.write(txt)
Pyto.print(txt, end="")
def init(args: argparse.Namespace):
con.clear()
con.box_print("Project Malcolm")
print("Local IP Address:", network.get_local_ip())
print("Public IP Address:", network.get_public_ip())
print("File size:", file.get_size(args.file), "Bytes")
print("Checksum:", file.gen_checksum(args.file))
print()
print("Press any key to start servers")
con.getch()
def read(text):
Pyto.print(text, end="")
def read(text):
console.print(text, end="")
def init():
"""
CLI Initialization demo.
"""
console.print(WELCOME_MESSAGE)
async def get_data(self) -> None:
total: int = 0
def _print(start, end, size):
nonlocal total
total += size
print(
f'\rProgress: {con.pretty_size(total)} / {con.pretty_size(self.file_size)},'
f'{con.pretty_size(size)}ps, Chunk[{start}/{end}]',
sep=' ',
end='')
def normalize(_tuple: Tuple[int, int],
start: int = 0) -> Sequence[str]:
# Return the data return by file.split to str to be sent as parameter
return [str(start + _tuple[0]), str(start + _tuple[1])]
async def fetch(start: int, end: int) \
-> MutableSequence[Tuple[Tuple[int, int], Optional[bytes]]]:
# Regenerate the connections
self.generate_connections()
# Split the data into parts for connections
_split: Sequence[Tuple[int, int]] = \
spilt(file_size=notnone(end - start), parts=len(self.ports))
# Fetch the data for each connection
_data: Sequence[Optional[bytes]] = [
# Assert if the type of element is bytes or None
assertoptionaltype(bytes, data) for data in
# Use high-level co-routine to fetch data
await asyncio.gather(
*(self._async_get(soc,
Request.TRANSFER,
normalize(tp, start),
progress=_print,
decode=False)
for soc, tp in zip(notnone(self.conns), _split)))
]
return list(zip(_split, _data))
async def verify(list_: MutableSequence[Tuple[Tuple[int, int], Optional[bytes]]]) \
-> Sequence[bytes]:
# Enumerate is required since we are updating the list contents
# Basically compensating for pass-by-value
for i, ((start, end), seg) in enumerate(list_):
try:
# If the segment is None
if seg is None:
list_[i] = (
(start, end),
bytes(b"".join(
# Assert if all the contents of sequence are bytes
# Recursive call itself until the content has been resolved
# Or all the connections have been refused
assertsequencetype(
bytes,
flatten_bytes(await
verify(await
fetch(start, end))),
))))
except AssertionError:
# If this exception occurs, all connections have been closed
await self.get_data()
# This should never be returned since self-call will quit the program
return []
return [notnone(seg) for (_, _), seg in list_]
# Check if there is resume data
if self.resume_data is not None:
# Load the resume data into tmp variable
self.data_unfinished = self.resume_data.data
else:
# Store the data in unverified (error-prone) data in tmp variable
self.data_unfinished = await fetch(0, notnone(self.file_size))
# Verify the data integrity
self.data = await verify(self.data_unfinished)
print()
def _print(start, end, size):
nonlocal counter
counter += 1
print(f'\rProgress: {counter}/{len(self.ports)}', end='')
def show():
"""
Generic sub-command to show a message.
"""
console.print("Get started with your CLI in no time!")
def __init__(self, *, address: str, ports: List[str], output: str,
resume: bool):
self.output = output
self.address = address
self.ports = ports
self.resume = resume
self.resume_data = None
if resume:
with open('resume.pyb', 'rb') as f:
self.resume_data = load(f)
# Show launch message to user
self.on_create()
# Get the checksum of the files
print('Fetching checksums ...')
asyncio.run(self.get_checksum())
# Verify files and remove the unmatched servers
self.verify_checksum()
# Fetch the file name from any server
print('Getting filename ... ')
self.get_file_name()
# Fetch the file size from any server
print('Getting filesize ...')
self.get_file_size()
# Get the data from server
print('Fetching data ...')
asyncio.run(self.get_data())
# Output data
print('Flushing data ...')
self.flush_data()
print('Done')
