def dumpDotFile(self, fp: TextIO) -> None:
"""
Creates a file that could be visualised using graphviz's DOT program.
Args:
fp(typing.TextIO): A text file stream that can be written to.
"""
self._nodeDict = dict()
transitionList: List[Tuple[str, str]] = list()
for r in self._roots:
if r is not None:
self._bfsAndAdd(r, transitionList)
fileContent: List[str] = ["digraph \""]
fileContent.append(self._id)
fileContent.append("\" {\n")
for transition in transitionList:
fileContent.append("\t\"")
fileContent.append(transition[0])
fileContent.append("\" -> \"")
fileContent.append(transition[1])
fileContent.append("\";\n")
fileContent.append("}\n")
fileText = "".join(fileContent)
fp.write(fileText)
fp.flush()
return
def make_archive(out: TextIO, sourcefp: str):
destfp = target + sourcefp[len(source):]
destfolder, _ = os.path.split(destfp)
try:
os.mkdir(destfolder)
except:
pass
print(sourcefp, '->', end=' ')
if os.path.exists(destfp + '.rar'):
print('Rar exists')
out.write("-1;{sourcefp}\n")
else:
process = subprocess.run([rar, 'a', '-ep', destfp + '.rar', sourcefp],
stdout=subprocess.PIPE,
universal_newlines=True,
encoding='cp437')
print(process.returncode)
out.write(f'{process.returncode};{sourcefp}\n')
out.flush()
backupfp = os.path.join(backup, sourcefp[len(source) + 1:])
backup_folder, _ = os.path.split(backupfp)
if not os.path.exists(backup_folder):
os.mkdir(backup_folder)
try:
os.rename(sourcefp, backupfp)
except Exception as ex:
msg = f' *** Failed to move folder {sourcefp} to {backup}: {ex}'
print(msg)
out.write(msg + '\n')
out.flush()
def display_progress(current: int,
total: int,
unit: Tuple[str, int, int],
output: TextIO = sys.stdout,
size: int = 100) -> None:
"""
Displays a progress bar in a text output stream
# Arguments
current: int. the current value of the progress bar
total: int. the max value of the progress bar
unit: Tuple[str, int, int]. the unit the progress is measured at:
(unit measurement string, unit measurement factor, number of digits after comma)
output: TextIO. the output stream to write the progress bar to (stdout by default)
size: int. the number of progression units on the progress bar
"""
done = int(size * current / total)
format_string = '\r[{:s}{:s}] ({:.' + str(unit[2]) + 'f} / {:.' + str(
unit[2]) + 'f}{:s})'
output.write(
format_string.format('█' * done, '.' * (size - done),
current / unit[1], total / unit[1], unit[0]))
output.flush()
def members_table(session,
members,
start_month,
end_month,
to: typing.TextIO = sys.stdout):
months = list(util.months(start_month, end_month))
members = members.all()
sys.stderr.flush()
to.flush()
to.write(' {name:^20} | '.format(name="Name"))
for month, next_month in months:
to.write(' {:2} '.format(month.month))
to.write('\n' + '-' * (20 + 3 + 5 * len(months)) + '\n')
for member in members:
to.write('{:2} '.format(member.id))
to.write('{:>20.20} | '.format(member.name))
for month, next_month in months:
fees = session.query(FeeEntry)\
.filter(FeeEntry.member_id == member.id)\
.filter(FeeEntry.month == month).all()
if len(fees) == 0:
to.write(' ')
else:
to.write(' {:>2.0f}{:1} '.format(
sum([f.fee for f in fees]),
'' if len(fees) == 1 else len(fees)))
to.write('\n')
def output_stats(
out_stream: TextIO,
name: str,
run_id: str,
metrics: Dict[str, Any],
config: Dict[str, Any],
):
for pass_name, metric in metrics.items():
logger.info(f"pass: {pass_name}")
for metric_name, records in metric.items():
total = 0
avg = 0
if records:
total = sum(records)
avg = total / len(records)
if metric_name.endswith(".time"):
logger.info(
f"metric: {metric_name}, average: {avg:.3f} ms, total: {total:.3f} ms"
)
logger.info(f"{format_float_val_list(records, 3)}")
elif metric_name.endswith(".memory"):
logger.info(
f"metric: {metric_name}, average: {avg:.3f} MB, total: {total:.3f} MB"
)
logger.info(f"{format_float_val_list(records, 3)}")
stats = {
"op_name": name,
"id": run_id,
"metric": metrics,
"config": config,
}
out_stream.write(json.dumps(stats) + "\n")
out_stream.flush()
async def write_handshake_response(
*,
file: typing.TextIO,
core_protocol_version: int = settings.CORE_PROTOCOL_VERSION,
protocol_version: int,
port: int,
certificate,
):
"""
Write the protocol versions, service name, address and certificate to the IO
for the client.
"""
certificate_der = utils.encode_certificate_der(certificate)
certificate_b64 = base64.b64encode(certificate_der).decode("ascii").rstrip(
"=")
file.write("{}|{}|{}|{}|{}|{}\n".format(
core_protocol_version,
protocol_version,
"tcp",
f"127.0.0.1:{port}",
"grpc",
certificate_b64,
))
file.flush()
def read_pwd(fd: TextIO, testing: TextIO, save: TextIO):
test_set = defaultdict(int)
for line in testing:
line = line.strip("\r\n")
test_set[line] += 1
for line in fd:
line = line.strip("\r\n")
items = [itm for itm in line.split("\t") if len(itm) > 0]
if len(items) == 2:
pwd = items[0]
if pwd not in test_set:
continue
struct = items[1]
terminals = terminal_re.findall(struct)
start = 0
lst = [pwd, f"{test_set[pwd]}"]
for terminal in terminals:
# tag = terminal[0]
num = int(terminal[1:])
segment = pwd[start:start + num]
start += num
lst.append(segment)
lst.append(terminal)
pass
save.write("\t".join(lst) + "\n")
pass
fd.close()
save.flush()
save.close()
pass
def handle_client(client_sock, replication_log: Queue, kv: dict,
kv_log: TextIO) -> None:
while True:
n = read_uint32(client_sock)
if n is None:
print(f"{current_thread().name} DONE")
break
data = read_bytes(client_sock, n).decode("utf-8")
print(f"{current_thread().name} {n} | {data}")
if data.startswith("set "):
_, key, val = data.split(" ")
kv[key] = val
# TODO - append to log
send_str(client_sock, "OK")
kv_log.write(data)
kv_log.write("\n")
kv_log.flush()
replication_log.put(data)
elif data.startswith("get "):
_, key = data.split(" ")
val = kv.get(key)
send_str(client_sock, val)
else:
send_str(client_sock, "💩")
print("Unknown command:", data)
def usage(fd: TextIO = stderr) -> None:
"""
usage(fd: TextIO = stderr) -> None
Print usage information to the specified descriptor.
"""
fd.write(__doc__.format(argv0=argv[0]))
fd.flush()
def _launch_redis(
port: int, stdout: TextIO = sys.stdout, stderr: TextIO = sys.stderr
) -> subprocess.Popen:
stdout.write(f">>>>>>>> Starting log for redis-server on port {port}\n")
stdout.flush()
try:
# inline config for redis server.
redis_process = subprocess.Popen(
[
"redis-server",
"--port",
f"{port}",
"--timeout",
"2",
"--loglevel",
"debug",
"--protected-mode",
"no",
],
stdout=stdout,
stderr=stderr,
)
except FileNotFoundError:
raise OSError(
"In order for redis to launch you need both the redis client and the python bindings. "
"You seem to be missing the redis client. Do 'conda install redis' and try again. If "
"failures continue you may need to download redis yourself, make it and add it to PATH."
)
atexit.register(redis_process.kill)
return redis_process
def monte_carlo_wrapper(rule: str, target: TextIO, save2: TextIO, n: int = 100000):
print(f"rule: {rule}", file=sys.stderr)
print(f"target: {target.name}", file=sys.stderr)
pcfg_scorer = MyScorer(rule=rule)
# sampling n passwords
rand_pairs = pcfg_scorer.gen_n_rand_pwd(n=n)
# generate corresponding rank list
minus_log_prob_list, ranks = gen_rank_from_minus_log_prob(rand_pairs)
del rand_pairs
# scoring passwords in test set
scored_pwd_list = pcfg_scorer.calc_minus_log2_prob_from_file(passwords=target)
target.close()
del pcfg_scorer
cracked = 0
prev_rank = 0
total = sum([n for n, _ in scored_pwd_list.values()])
# estimating
print("Estimating...", file=sys.stderr)
for pwd, info in sorted(scored_pwd_list.items(), key=lambda x: x[1][1], reverse=False):
num, mlp = info
# rank should be an integer, and larger than previous one
rank = ceil(max(minus_log_prob2rank(minus_log_prob_list, ranks, mlp), prev_rank + 1))
prev_rank = rank
cracked += num
save2.write(f"{pwd}\t{mlp:.8f}\t{num}\t{rank}\t{cracked}\t{cracked / total * 100:.2f}\n")
save2.flush()
save2.close()
del minus_log_prob_list
del ranks
del scored_pwd_list
def set_scroll_region(nrows: int, file: TextIO):
"""Set terminal scroll region sizein terms of rows."""
print('Set scroll region to', nrows)
file.write('\n')
with ancurs(file):
file.write(seq.ScrollRegion.SET.format(0, nrows - 1))
file.write(seq.Cursor.goes.UP)
file.flush()
def count(self, fp: TextIO) -> None:
with self.server.queue.lock:
crit = 0
for n in self.server.queue:
if n.urgency == Urgency.CRITICAL:
crit += 1
fp.write(f'{len(self.server.queue)},{crit}')
fp.flush()
def _print(destination: TextIO, message='', end='\n', flush=False, lock=True) -> None:
if lock:
PrintLock().__enter__()
destination.write(f'{message}{end}')
if flush:
destination.flush()
if lock:
PrintLock().__exit__()
def print_model(simulation: Iterator[ActionPairState[ModelActionPair, ModelActionPair]],
output: TextIO=sys.stdout) \
-> Generator[ActionPairState[ModelActionPair, ModelActionPair], None, None]:
for s in simulation:
output.write("Model Parameters were: ({},{})\n".format(
s.get_last_x_action().get_model(),
s.get_last_y_action().get_model()))
output.flush()
yield s
def save(data: Generator[Tuple[str, float, int, int, int, float], Any, None],
fd: TextIO,
close_fd: bool = True):
for pwd, prob, cnt, rank, cracked, cracked_rate in data:
fd.write(
f"{pwd}\t{prob}\t{cnt}\t{rank}\t{cracked}\t{cracked_rate:5.2f}\n")
fd.flush()
if close_fd:
fd.close()
def main(input_file: typing.TextIO = sys.stdin,
output_file: typing.TextIO = sys.stdout) -> None:
try:
output_file.write("What is your name? ")
output_file.flush()
name = input_file.readline()[:-1]
output_file.write(f"Hi {name}, welcome to the team!\n")
except KeyboardInterrupt:
pass
def _output(level: int, fp: TextIO, header: Text, color: Text,
message: object):
if level < LOG_LEVEL:
return
proc_name = '' if PROC_NAME is None else f'/"{PROC_NAME}"'
if fp.isatty():
fp.write(f'{color}({header}{proc_name}){Fore.RESET} {message}\n')
else:
fp.write(f'({header}{proc_name}) {message}\n')
fp.flush()
def print_distributions(simulation: Iterator[ActionPairState[ModelActionPair[Any, ActionDistributionPair],
ModelActionPair[Any, ActionDistributionPair]]],
output: TextIO=sys.stdout) \
-> Generator[ActionPairState[ModelActionPair[Any, ActionDistributionPair],
ModelActionPair[Any, ActionDistributionPair]], None, None]:
for s in simulation:
output.write("Action Distributions were: ({},{})\n".format(
s.get_last_x_action().get_action().get_distribution(),
s.get_last_y_action().get_action().get_distribution()))
output.flush()
yield s
def print_count(
simulation: Iterator[ActionPairState],
output: TextIO = sys.stdout) -> Generator[ActionPairState, None, None]:
count = 0
for s in simulation:
output.write("Starting simulation round: {}\n".format(count))
output.flush()
yield s
count += 1
output.write("\n")
output.flush()
def _tee_pipe_run(self, pipe: IO[bytes], stream: TextIO,
file_name: pathlib.Path) -> None:
with open(file_name, "wb") as file:
while True:
# We want the output to be interactive, so we make sure to
# immediately write out every byte we receive. However, we
# still rely on any write buffering used by the file and
# stream.
data = pipe.read(1)
if len(data) == 0:
break
file.write(data)
stream.write(data.decode(stream.encoding))
stream.flush()
def handle(self, stdin: TextIO, stdout: TextIO, stderr: TextIO) -> None:
time_limit = int(stdin.readline().strip())
player = stdin.readline().strip()
board = stdin.readline().strip()
with redirect_stdout(sys.stderr if self.logging else None):
move, err, time_limit = self.get_move(board, player, time_limit)
if err is not None:
stderr.write(f"SERVER: {err}\n")
stdout.write(f"{move};{time_limit}\n")
stdout.flush()
stderr.flush()
async def fetch(session: aiohttp.ClientSession,
semaphore: asyncio.BoundedSemaphore, pbar: tqdm, url: str,
failed: TextIO):
async with semaphore:
async with session.get(url) as response:
status = response.status
content = await response.content.read()
if status != 200:
failed.write(url + '\n')
failed.flush()
return
fn = url.rsplit("/")[-1]
with open(fn, 'wb') as out:
out.write(content)
pbar.update(1)
def __buffered_reader(stdread: paramiko.ChannelFile, stdwrite: TextIO):
global SIGINT
import select
import time
channel: paramiko.Channel = stdread.channel
while not SIGINT and not channel.exit_status_ready():
if channel.recv_ready():
r, w, x = select.select([channel], [], [], 0.0)
if len(r) > 0:
stdwrite.buffer.write(channel.recv(1024))
stdwrite.flush()
else:
time.sleep(0.2)
SIGINT = True
def analyse_archives(out: TextIO):
totf = 0
for fullpath in get_all_files(source):
totf += 1
path, file = os.path.split(fullpath)
basename, ext = os.path.splitext(file)
if ext.lower() in ['.cbr', '.rar']:
print(f'{file:<100}', end='')
try:
status = 'Ok'
nf, ni, np, na, no, nm, so = analyze_one_archive(fullpath)
except:
status = 'Error'
nf, ni, np, na, no, nm, so = (0, 0, 0, 0, 0, 0, '')
print(status, nf, ni, np, na, no, nm, so, end='')
if status == 'Error':
move(fullpath, "error")
else:
if so.lower().find(".exe") >= 0:
move(fullpath, "exe")
else:
if ni == 0 and na == 0 and np == 1:
move(fullpath, "pdf1")
else:
if ni == 0 and na == 0 and np > 1:
move(fullpath, "pdf")
else:
if nf == 1 and np == 0 and na == 0 and ni > 0:
move(fullpath, "cbr1")
else:
if nf >= 1 and np == 0 and na == 0 and ni > 0:
move(fullpath, "cbrn")
else:
if ni == 0 and np == 0 and na == 1:
move(fullpath, "archive1")
else:
if ni == 0 and np == 0 and na > 1:
move(fullpath, "archiven")
else:
move(fullpath, "hybrid")
out.write(f'{dest};{status};{file};{nf};{ni};{np};{na};{no};{nm};{so}\n')
out.flush()
if not DO_IT:
print("No action: ", end='')
print(f'{totf} fichiers analysés')
def write_stream_with_colors_win_py3(stream: 'BaseStream', outfile: TextIO,
flush: bool):
"""Like `write`, but colorized chunks are written as text
directly to `outfile` to ensure it gets processed by colorama.
Applies only to Windows and colorized terminal output.
"""
color = b'\x1b['
encoding = outfile.encoding
for chunk in stream:
if color in chunk:
outfile.write(chunk.decode(encoding))
else:
outfile.buffer.write(chunk)
if flush:
outfile.flush()
def send_mail (self, print_mail: bool = False, outstream: TextIO = sys.stdout) -> bool: """Send the mail or print it, if ``print_mail'' is True to ``outstream''""" if self.sender is not None: self.email.set_from (self.sender) for r in self.recv: self.email.add_to (r) if self.cc: for r in self.cc: self.email.add_cc (r) if print_mail: outstream.write (self.email.build_mail ()) outstream.flush () status = True else: (status, rc, out, err) = self.email.send_mail () return status
def _repl(self, input: TextIO, output: TextIO):
"""the read eval print loop of the daemon.
Waits for user input received by calling the exposed function of the daemon"""
runLoop = True
print("reached repl!")
while runLoop:
print("looped!")
command = []
while runLoop:
line = input.readline()
if line[-3:] == DONE:
command.extend(line[:-3].strip().lower().split()
) #The writer is leaving.
break
if line == '': #writers gone.
command = []
input.close()
input = open(daemonInput,
'r') #locks until there's a writer
continue
line = line.strip().lower()
if line != '':
command.extend(line.split())
if len(command) == 0:
pass
else:
if command[0] == "attach":
output.write("press enter to detach.")
time.sleep(1)
self.attach_guard, self.cec.stdout = self.cec.stdout, output
output.write(DONE)
elif command[0] == "detach":
self.cec.stdout = self.attach_guard
output.write(ATTACHDONE)
output.flush()
elif command[0] in self.cec.interactive_cmd:
out_guard = self.cec.stdout
self.cec.stdout = output # switch the output sink just for this command execution time
self.cec.interactive_cmd[command[0]](self.cec,
*command[1:])
self.cec.stdout = out_guard
else:
print(
"Unknown command.\n Use'--help' argument for a list of available commands",
file=output)
output.write(DONE)
output.flush()
def write_message(
stream: TextIO,
message: str,
end: str = '\n',
) -> None:
"""
Write a message on the stream.
Args:
stream: a stream
message: a message
end: a string appended after the message, default a newline
"""
stream.write(message)
if end:
stream.write(end)
stream.flush()
def save(uniq_lines: Set[str], save2: TextIO, order: str):
if not save2.writable() or save2.closed:
print(f"{save2.name} can not be used to write")
sys.exit(-1)
if order == 'order':
uniq_lines = sorted(uniq_lines, reverse=False)
elif order == 'reverse':
uniq_lines = sorted(uniq_lines, reverse=True)
elif order == 'random':
uniq_lines = list(uniq_lines)
shuffle(uniq_lines)
else:
sys.stderr.write(f"Unknown method: {order}")
sys.exit(-1)
for _l in uniq_lines:
save2.write(f"{_l}\n")
save2.flush()
