def variable_catalog(schema_basepath: str, schema_name: str,
fh: TextIO) -> None:
schema: Optional[Schema] = Schema.load(schema_name,
base_path=schema_basepath)
assert schema is not None
write_catalog(schema, fh)
fh.close()
def read_entries(file: TextIO):
entries = set()
for line in file:
line = line.strip("\r\n")
entries.add(line.lower())
file.close()
return entries
def write(self, oglObject: OglClass, file: TextIO):
"""
Write data to filename.
Format:
```python
Class Name
<<stereotype_optional>>
+method([param[:type]]*)[:type_return]
+field[:type][=value_initial]
```
Args:
oglObject: The Ogl object to edit
file: The text file to write to
"""
o: PyutClass = cast(PyutClass, oglObject.getPyutObject())
file.write(o.getName() + osLineSep)
if o.getStereotype() is not None:
file.write(str(o.getStereotype()) + osLineSep)
for method in o.methods:
file.write(method.getString() + osLineSep)
for field in o.fields:
file.write(str(field) + osLineSep)
file.close()
def poll_subprocess(self, log_file: TextIO):
""" Wait for ffmpeg subprocess to end; close log file and set the recording state. """
while self.ffmpeg_process.poll() is None:
pass
log_file.close()
self.ffmpeg_process = None
self.set_recording_state(False)
def wrapper(hc_res: TextIO, f_target: TextIO, save2: TextIO, wanted_indices: List[int], guess_number_idx: int):
wanted_pos = read_hc_res(hc_res, wanted_indices=wanted_indices, guess_number_idx=guess_number_idx)
hc_res.close()
target_cnt = read_target_set(f_target)
f_target.close()
hcg = hcgood(wanted_pos, target_cnt)
save(hcg, save2, close_fd=True)
def wrapper(hc_res: TextIO, targets: TextIO, save2: TextIO):
pwd_pos = read_hc_res(hc_res)
hc_res.close()
pwd_cnt = read_target_set(targets)
targets.close()
hcg = hcgood(pwd_pos, pwd_cnt)
save(hcg, save2, close_fd=True)
def read_pwd_cnt(fd: TextIO, close_fd: bool = True):
pwd_cnt = defaultdict(int)
for line in fd:
pwd_cnt[line.strip("\r\n")] += 1
if close_fd:
fd.close()
return pwd_cnt
def save_data(data: RulesData, output: TextIO) -> None:
"""save_data(data: RulesData, str, str], output: TextIO)
Saves the data from parsing the accession description page to a JSON format file. output must be a
file open for writing."""
json.dump(data, output, indent=2)
output.close()
def shutdown_log(ret_code: int, log_file: TextIO) -> None:
"""Closes log with last session dump if necessary"""
if is_unexpected_exit(ret_code):
dump_last_log_session(log_file)
log_file.close()
def main(csv_file: TextIO, output_dir: Path) -> int:
output_dir.mkdir(exist_ok=True)
dialect = csv.Sniffer().sniff(csv_file.read(1024 * 4))
csv_file.seek(0)
reader = csv.DictReader(csv_file, dialect=dialect)
rows = list(reader)
csv_file.close()
def is_question(q: str) -> bool:
return q not in ["Timestamp", "Email address"]
questions = filter(is_question, rows[0].keys())
data = {q: get_data(rows, q) for q in questions}
max_n = max(max(v[1]) for v in data.values())
print(max_n)
y_ticks = np.arange(0, max_n + 2, step=2)
for question, counter in data.items():
filename = safe_filename(question) + ".png"
filename = output_dir / filename
create_graph(counter, question, filename, y_ticks=y_ticks)
return 0
def read_test(fd_test: TextIO):
pwd_cnt = defaultdict(int)
for line in fd_test:
line = line.strip()
pwd_cnt[line] += 1
fd_test.close()
return pwd_cnt
def read_pwd(fd: TextIO, testing: TextIO, save: TextIO):
test_set = defaultdict(int)
for line in testing:
line = line.strip("\r\n")
test_set[line] += 1
for line in fd:
line = line.strip("\r\n")
items = [itm for itm in line.split("\t") if len(itm) > 0]
if len(items) == 2:
pwd = items[0]
if pwd not in test_set:
continue
struct = items[1]
terminals = terminal_re.findall(struct)
start = 0
lst = [pwd, f"{test_set[pwd]}"]
for terminal in terminals:
# tag = terminal[0]
num = int(terminal[1:])
segment = pwd[start:start + num]
start += num
lst.append(segment)
lst.append(terminal)
pass
save.write("\t".join(lst) + "\n")
pass
fd.close()
save.flush()
save.close()
pass
def display(results: Results, json_file: TextIO):
if json_file:
# write to the file
print("We're writing to a JSON File")
json.dump(
{
"successful_requests": results.successful_requests(),
"slowest": results.slowest(),
"fastest": results.fastest(),
"total_time": results.total_time,
"requests_per_minute": results.requests_per_minute(),
"requests_per_second": results.requests_per_second(),
},
json_file,
)
json_file.close()
print(".... Done!")
else:
# Print to screen
print(".... Done!")
print("--- Results ---")
print(f"Successful requests\t{results.successful_requests()}")
print(f"Slowest \t{results.slowest()}")
print(f"Fastest \t{results.fastest()}")
print(f"Total Time \t{results.total_time}")
print(f"Requests Per Minute\t{results.requests_per_minute()}")
print(f"Requests Per Second\t{results.requests_per_second()}")
def display(results: Results, json_file: TextIO):
if json_file:
print("We are writing to a JSON file")
json.dump(
{
"successful_requests": results.successful_requests(),
"slowest": results.slowest(),
"fastest": results.fastest(),
"total_time": results.total_time,
"requests per minute": results.requests_per_minute(),
"requests per second": results.requests_per_second(),
},
json_file,
)
json_file.close()
print("file copy done")
else:
print("......Done")
print("----Results ----")
print(f"successful requets \t{results.successful_requests()}")
print(f"slowest requets \t{results.slowest()}s")
print(f"fastest requets \t{results.fastest()}s")
print(f"Total time \t{results.total_time}")
print(f"requets per minute \t{results.requests_per_minute()}")
print(f"requets per second \t{results.requests_per_second()}")
def monte_carlo_wrapper(rule: str, target: TextIO, save2: TextIO, n: int = 100000):
print(f"rule: {rule}", file=sys.stderr)
print(f"target: {target.name}", file=sys.stderr)
pcfg_scorer = MyScorer(rule=rule)
# sampling n passwords
rand_pairs = pcfg_scorer.gen_n_rand_pwd(n=n)
# generate corresponding rank list
minus_log_prob_list, ranks = gen_rank_from_minus_log_prob(rand_pairs)
del rand_pairs
# scoring passwords in test set
scored_pwd_list = pcfg_scorer.calc_minus_log2_prob_from_file(passwords=target)
target.close()
del pcfg_scorer
cracked = 0
prev_rank = 0
total = sum([n for n, _ in scored_pwd_list.values()])
# estimating
print("Estimating...", file=sys.stderr)
for pwd, info in sorted(scored_pwd_list.items(), key=lambda x: x[1][1], reverse=False):
num, mlp = info
# rank should be an integer, and larger than previous one
rank = ceil(max(minus_log_prob2rank(minus_log_prob_list, ranks, mlp), prev_rank + 1))
prev_rank = rank
cracked += num
save2.write(f"{pwd}\t{mlp:.8f}\t{num}\t{rank}\t{cracked}\t{cracked / total * 100:.2f}\n")
save2.flush()
save2.close()
del minus_log_prob_list
del ranks
del scored_pwd_list
def samp(corpus: TextIO, samp_corpora: List[TextIO],
samp_size: int, fd_removed: TextIO, valid_pwd: Pattern):
for samp_corpus in samp_corpora:
if not samp_corpus.writable():
print("Training and Testing SHOULD be Writable!", file=sys.stderr)
sys.exit(-1)
if len(samp_corpora) < 1:
print("At least one sample file!", file=sys.stderr)
sys.exit(-1)
pwd_set = []
count_invalid = defaultdict(int)
for line in corpus:
line = line.strip("\r\n")
if valid_pwd.match(line) is None:
count_invalid[line] += 1
continue
pwd_set.append(line)
samp_size = min(len(pwd_set), samp_size)
for idx, samp_corpus in enumerate(samp_corpora):
shuffle(pwd_set)
for line in pwd_set[:samp_size]:
samp_corpus.write(f"{line}\n")
samp_corpus.flush()
print(f"{idx + 1} sample file saved here: {samp_corpus.name}", file=sys.stderr)
samp_corpus.close()
if len(count_invalid) != 0 and fd_removed is not None:
print(f"Removed invalid passwords saved in {fd_removed.name}", file=sys.stderr)
for p, n in sorted(count_invalid.items(), key=lambda x: x[1], reverse=True):
fd_removed.write(f"{p}\t{n}\n")
fd_removed.close()
print("Done!", file=sys.stderr)
def display(results: Results, json_file: TextIO):
if json_file:
# Write to a file
json.dump(
{
"successful_requests": results.successful_requests(),
"slowest": results.slowest(),
"fastest": results.fastest(),
"total_time": results.total_time,
"requests_per_minute": results.requests_per_minute(),
"requests_per_second": results.requests_per_second(),
},
json_file,
)
json_file.close()
print(".... Done!")
else:
print(".... Done!")
print("--- Results ---")
print(f"Successful requests\t{results.successful_requests()}")
print(f"Slowest \t{results.slowest()}")
print(f"Fastest \t{results.fastest()}")
print(f"Average time \t{results.average_time()}")
print(f"Requests per minute\t{results.requests_per_minute()}")
print(f"Requests per second\t{results.requests_per_second()}")
def main(infile: TextIO) -> int:
scores: Iterator[int] = map(int, infile.read().split())
infile.close()
counter = Counter(scores)
n = sum(v for v in counter.values())
detractors = sum(v for k, v in counter.items() if k < 7)
promoters = sum(v for k, v in counter.items() if k > 8)
p_detractors = (detractors / n) * 100
p_promoters = (promoters / n) * 100
nps = p_promoters - p_detractors
for k in range(11):
print(f"{k}: {counter.get(k, 0)}")
print("\n")
print(f"n= {n}")
print(f"n_detractors= {detractors} ({p_detractors}%)")
print(f"n_promoters= {promoters} ({p_promoters}%)")
print(f"NPS= {nps}")
return 0
def database_edit(ctx: Context, database: Callable[..., Database], table: str, _id: str | int, file: TextIO,
submission_file: BytesIO | None, submission_thumbnail: BytesIO | None):
"""
Edit entries and submission files manually using a JSON file. Submission files/thumbnails can be added using the
respective options.
The JSON fields must match the column names of the selected table. For a list of columns for each table, please see
the README at {blue}https://pypi.org/project/{prog_name}/{version}{reset}.
If the {yellow}--submission-file{reset} and/or {yellow}--submission-thumbnail{reset} options are used, the
{yellow}FILE{reset} argument can be omitted.
"""
db: Database = database()
db_table: Table = get_table(db, table)
data: dict = load(file)
file.close()
if not data and table.lower() != db.submissions.name.lower():
raise BadParameter(f"FILE cannot be empty for {table} table.", ctx, get_param(ctx, "file"))
elif not isinstance(data, dict):
raise BadParameter(f"Data must be in JSON object format.", ctx, get_param(ctx, "file"))
add_history(db, ctx, table=table, id=_id, file=file.name,
submission_file=submission_file.name if submission_file else None,
submission_thumbnail=submission_thumbnail.name if submission_thumbnail else None)
if (entry := db_table[_id]) is None:
raise BadParameter(f"No entry with ID {_id} in {table}.", ctx, get_param(ctx, "_id"))
def display(input: TextIO) -> None:
import seaborn as sns
# Load view JSON.
input_data: dict = json.load(input)
input.close()
loader = PackageLoader('mockdown.display', 'templates')
env = Environment(loader=loader,
autoescape=select_autoescape(['html', 'xml']))
# Kind of a hack. Using importlib.resources is probably more Pythonic but...
kiwi_js_src = loader.get_source(env, 'js/flightlessbird.all.js')[0]
input_meta = input_data.get('meta', {})
scrape_meta = input_data.get('scrape', {})
# Utility for convenience.
# todo: move elsewhere
def get_meta(*keys: str, subject=input_meta, default=None):
if len(keys) == 0:
return input_meta
k, ks = keys[0], keys[1:]
result = input_meta.get(k, default)
if not ks:
return result
else:
return get_meta(*ks, subject=result, default=default)
examples = input_data['examples']
observed_bounds = {
'min_width': min(e['rect'][2] - e['rect'][0] for e in examples),
'max_width': max(e['rect'][2] - e['rect'][0] for e in examples),
'min_height': min(e['rect'][3] - e['rect'][1] for e in examples),
'max_height': max(e['rect'][3] - e['rect'][1] for e in examples),
}
context = {
'examples': examples,
'origin': get_meta('scrape', 'origin', default=None),
'colors': sns.color_palette('bright', len(examples)).as_hex(),
**observed_bounds
}
template = env.get_template('default.html.jinja2')
html = template.render(**context)
# Write the result to a temporary file, and open it in the user's web browser.
tmp = tempfile.NamedTemporaryFile(mode='w',
prefix='fnord-',
suffix='.html',
delete=False)
tmp.write(html)
tmp.flush()
tmp.close()
webbrowser.open(f"file://{tmp.name}")
def read_raw_data(fd: TextIO,
skip=1,
splitter=re.compile("\t"),
idx_pwd=0,
idx_rank=1,
idx_num=2):
lst = []
for _ in range(skip):
fd.readline()
for line in fd:
line = line.strip("\r\n")
items = splitter.split(line)
pwd = items[idx_pwd]
rank = float(items[idx_rank])
num = int(items[idx_num])
lst.append((pwd, rank, num))
fd.close()
lst = sorted(lst, key=lambda x: x[1])
num_list = [n for _, _, n in lst]
total = sum(num_list)
top_25 = floor(0.25 * total)
bottom_25 = ceil(0.75 * total)
easiest_items = [p for p, _, _ in lst[:top_25]]
hardest_items = [p for p, _, _ in lst[bottom_25:]]
return set(easiest_items), set(hardest_items), total
def __init__(self,
json_file: TextIO,
close_fd: bool,
use_rate: bool = True):
data = json.load(json_file)
y_list = data["y_list"]
total = data["total"]
if use_rate:
y_list = [cracked / total * 100 for cracked in y_list]
if close_fd:
json_file.close()
elif json_file.seekable():
json_file.seek(0)
self.x_list = data["x_list"]
self.y_list = y_list
self.color = data['color']
self.marker = data['marker']
self.marker_size = data['marker_size']
self.mark_every = data["mark_every"]
self.line_width = data['line_width']
if type(data['line_style']) is str:
self.line_style = data['line_style']
else:
self.line_style = (data['line_style'][0],
tuple(list(data['line_style'][1])))
self.label = data['label']
self.text = data['label']
self.show_text = data['show_text']
self.text_x = data['text_x']
self.text_y = data['text_y']
self.text_fontsize = data['text_fontsize']
self.text_color = data['text_color']
def read_nn(fd_csv: TextIO):
pwd_dict = {}
for line in fd_csv:
lst = line.strip("\r\n").split('\t')
pwd, prob, guess_number = lst[0:3]
pwd_dict[pwd] = (float(prob), float(guess_number))
fd_csv.close()
return pwd_dict
def saver(itr: Generator[Tuple[str, float, int, int, int, float]],
save2: TextIO):
for pwd, ml2p, num, rank, cracked, cracked_rate in itr:
save2.write(
f"{pwd}\t{ml2p:.8f}\t{num}\t{rank}\t{cracked}\t{cracked_rate:.2f}\n"
)
save2.close()
pass
def redirect_stream(input_stream: TextIO,
output_stream: TextIO) -> None:
while True:
line = input_stream.readline()
if line == "":
input_stream.close()
return
output_stream.write(line)
def from_files(cls, schema_basepath: str, source_schema: str, target_schema: str, output_file: TextIO) -> None:
source_schema_instance: Optional[Schema] = Schema.load(source_schema, base_path=schema_basepath)
target_schema_instance: Optional[Schema] = Schema.load(target_schema, source_schema=source_schema_instance,
base_path=schema_basepath)
assert target_schema_instance is not None
export: "ExportLinkages" = cls(target_schema_instance, output_file)
export()
output_file.close()
def _read_file(input_file: TextIO) -> str:
is_stdin = input_file is sys.stdin
if is_stdin:
LOGGER.info("↓↓↓ Write your source code here. End with CTR^D ↓↓↓")
content = input_file.read()
if not is_stdin:
input_file.close()
return content
def freq(infile: TextIO, outfile: TextIO) -> None:
count = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] #list with 10 integers
# above can also use count = [0] * 10
for line in infile:
count[int(line)] += 1
infile.close()
for i in range(len(count)):
outfile.write(f'{i} {count[i]}\n')
outfile.close()
def try_to_close_output_file(self, output_file: TextIO):
if output_file:
try:
output_file.close()
logging.debug(f"Closed file {self.output_file}.")
except Exception as e:
logging.error(
f"Error when trying to close {self.output_file}: {e}")
sys.exit(EXIT_PROCESSING_ERROR)
def _read_file(input_file: TextIO) -> str:
if input_file is sys.stdin:
LOGGER.debug('Write your source code here. End with CTR^D')
content = input_file.read()
else:
content = input_file.read()
input_file.close()
return content
