def variable_catalog(schema_basepath: str, schema_name: str, fh: TextIO) -> None: schema: Optional[Schema] = Schema.load(schema_name, base_path=schema_basepath) assert schema is not None write_catalog(schema, fh) fh.close()
def read_entries(file: TextIO): entries = set() for line in file: line = line.strip("\r\n") entries.add(line.lower()) file.close() return entries
def write(self, oglObject: OglClass, file: TextIO): """ Write data to filename. Format: ```python Class Name <<stereotype_optional>> +method([param[:type]]*)[:type_return] +field[:type][=value_initial] ``` Args: oglObject: The Ogl object to edit file: The text file to write to """ o: PyutClass = cast(PyutClass, oglObject.getPyutObject()) file.write(o.getName() + osLineSep) if o.getStereotype() is not None: file.write(str(o.getStereotype()) + osLineSep) for method in o.methods: file.write(method.getString() + osLineSep) for field in o.fields: file.write(str(field) + osLineSep) file.close()
def poll_subprocess(self, log_file: TextIO): """ Wait for ffmpeg subprocess to end; close log file and set the recording state. """ while self.ffmpeg_process.poll() is None: pass log_file.close() self.ffmpeg_process = None self.set_recording_state(False)
def wrapper(hc_res: TextIO, f_target: TextIO, save2: TextIO, wanted_indices: List[int], guess_number_idx: int): wanted_pos = read_hc_res(hc_res, wanted_indices=wanted_indices, guess_number_idx=guess_number_idx) hc_res.close() target_cnt = read_target_set(f_target) f_target.close() hcg = hcgood(wanted_pos, target_cnt) save(hcg, save2, close_fd=True)
def wrapper(hc_res: TextIO, targets: TextIO, save2: TextIO): pwd_pos = read_hc_res(hc_res) hc_res.close() pwd_cnt = read_target_set(targets) targets.close() hcg = hcgood(pwd_pos, pwd_cnt) save(hcg, save2, close_fd=True)
def read_pwd_cnt(fd: TextIO, close_fd: bool = True): pwd_cnt = defaultdict(int) for line in fd: pwd_cnt[line.strip("\r\n")] += 1 if close_fd: fd.close() return pwd_cnt
def save_data(data: RulesData, output: TextIO) -> None: """save_data(data: RulesData, str, str], output: TextIO) Saves the data from parsing the accession description page to a JSON format file. output must be a file open for writing.""" json.dump(data, output, indent=2) output.close()
def shutdown_log(ret_code: int, log_file: TextIO) -> None: """Closes log with last session dump if necessary""" if is_unexpected_exit(ret_code): dump_last_log_session(log_file) log_file.close()
def main(csv_file: TextIO, output_dir: Path) -> int: output_dir.mkdir(exist_ok=True) dialect = csv.Sniffer().sniff(csv_file.read(1024 * 4)) csv_file.seek(0) reader = csv.DictReader(csv_file, dialect=dialect) rows = list(reader) csv_file.close() def is_question(q: str) -> bool: return q not in ["Timestamp", "Email address"] questions = filter(is_question, rows[0].keys()) data = {q: get_data(rows, q) for q in questions} max_n = max(max(v[1]) for v in data.values()) print(max_n) y_ticks = np.arange(0, max_n + 2, step=2) for question, counter in data.items(): filename = safe_filename(question) + ".png" filename = output_dir / filename create_graph(counter, question, filename, y_ticks=y_ticks) return 0
def read_test(fd_test: TextIO): pwd_cnt = defaultdict(int) for line in fd_test: line = line.strip() pwd_cnt[line] += 1 fd_test.close() return pwd_cnt
def read_pwd(fd: TextIO, testing: TextIO, save: TextIO): test_set = defaultdict(int) for line in testing: line = line.strip("\r\n") test_set[line] += 1 for line in fd: line = line.strip("\r\n") items = [itm for itm in line.split("\t") if len(itm) > 0] if len(items) == 2: pwd = items[0] if pwd not in test_set: continue struct = items[1] terminals = terminal_re.findall(struct) start = 0 lst = [pwd, f"{test_set[pwd]}"] for terminal in terminals: # tag = terminal[0] num = int(terminal[1:]) segment = pwd[start:start + num] start += num lst.append(segment) lst.append(terminal) pass save.write("\t".join(lst) + "\n") pass fd.close() save.flush() save.close() pass
def display(results: Results, json_file: TextIO): if json_file: # write to the file print("We're writing to a JSON File") json.dump( { "successful_requests": results.successful_requests(), "slowest": results.slowest(), "fastest": results.fastest(), "total_time": results.total_time, "requests_per_minute": results.requests_per_minute(), "requests_per_second": results.requests_per_second(), }, json_file, ) json_file.close() print(".... Done!") else: # Print to screen print(".... Done!") print("--- Results ---") print(f"Successful requests\t{results.successful_requests()}") print(f"Slowest \t{results.slowest()}") print(f"Fastest \t{results.fastest()}") print(f"Total Time \t{results.total_time}") print(f"Requests Per Minute\t{results.requests_per_minute()}") print(f"Requests Per Second\t{results.requests_per_second()}")
def display(results: Results, json_file: TextIO): if json_file: print("We are writing to a JSON file") json.dump( { "successful_requests": results.successful_requests(), "slowest": results.slowest(), "fastest": results.fastest(), "total_time": results.total_time, "requests per minute": results.requests_per_minute(), "requests per second": results.requests_per_second(), }, json_file, ) json_file.close() print("file copy done") else: print("......Done") print("----Results ----") print(f"successful requets \t{results.successful_requests()}") print(f"slowest requets \t{results.slowest()}s") print(f"fastest requets \t{results.fastest()}s") print(f"Total time \t{results.total_time}") print(f"requets per minute \t{results.requests_per_minute()}") print(f"requets per second \t{results.requests_per_second()}")
def monte_carlo_wrapper(rule: str, target: TextIO, save2: TextIO, n: int = 100000): print(f"rule: {rule}", file=sys.stderr) print(f"target: {target.name}", file=sys.stderr) pcfg_scorer = MyScorer(rule=rule) # sampling n passwords rand_pairs = pcfg_scorer.gen_n_rand_pwd(n=n) # generate corresponding rank list minus_log_prob_list, ranks = gen_rank_from_minus_log_prob(rand_pairs) del rand_pairs # scoring passwords in test set scored_pwd_list = pcfg_scorer.calc_minus_log2_prob_from_file(passwords=target) target.close() del pcfg_scorer cracked = 0 prev_rank = 0 total = sum([n for n, _ in scored_pwd_list.values()]) # estimating print("Estimating...", file=sys.stderr) for pwd, info in sorted(scored_pwd_list.items(), key=lambda x: x[1][1], reverse=False): num, mlp = info # rank should be an integer, and larger than previous one rank = ceil(max(minus_log_prob2rank(minus_log_prob_list, ranks, mlp), prev_rank + 1)) prev_rank = rank cracked += num save2.write(f"{pwd}\t{mlp:.8f}\t{num}\t{rank}\t{cracked}\t{cracked / total * 100:.2f}\n") save2.flush() save2.close() del minus_log_prob_list del ranks del scored_pwd_list
def samp(corpus: TextIO, samp_corpora: List[TextIO], samp_size: int, fd_removed: TextIO, valid_pwd: Pattern): for samp_corpus in samp_corpora: if not samp_corpus.writable(): print("Training and Testing SHOULD be Writable!", file=sys.stderr) sys.exit(-1) if len(samp_corpora) < 1: print("At least one sample file!", file=sys.stderr) sys.exit(-1) pwd_set = [] count_invalid = defaultdict(int) for line in corpus: line = line.strip("\r\n") if valid_pwd.match(line) is None: count_invalid[line] += 1 continue pwd_set.append(line) samp_size = min(len(pwd_set), samp_size) for idx, samp_corpus in enumerate(samp_corpora): shuffle(pwd_set) for line in pwd_set[:samp_size]: samp_corpus.write(f"{line}\n") samp_corpus.flush() print(f"{idx + 1} sample file saved here: {samp_corpus.name}", file=sys.stderr) samp_corpus.close() if len(count_invalid) != 0 and fd_removed is not None: print(f"Removed invalid passwords saved in {fd_removed.name}", file=sys.stderr) for p, n in sorted(count_invalid.items(), key=lambda x: x[1], reverse=True): fd_removed.write(f"{p}\t{n}\n") fd_removed.close() print("Done!", file=sys.stderr)
def display(results: Results, json_file: TextIO): if json_file: # Write to a file json.dump( { "successful_requests": results.successful_requests(), "slowest": results.slowest(), "fastest": results.fastest(), "total_time": results.total_time, "requests_per_minute": results.requests_per_minute(), "requests_per_second": results.requests_per_second(), }, json_file, ) json_file.close() print(".... Done!") else: print(".... Done!") print("--- Results ---") print(f"Successful requests\t{results.successful_requests()}") print(f"Slowest \t{results.slowest()}") print(f"Fastest \t{results.fastest()}") print(f"Average time \t{results.average_time()}") print(f"Requests per minute\t{results.requests_per_minute()}") print(f"Requests per second\t{results.requests_per_second()}")
def main(infile: TextIO) -> int: scores: Iterator[int] = map(int, infile.read().split()) infile.close() counter = Counter(scores) n = sum(v for v in counter.values()) detractors = sum(v for k, v in counter.items() if k < 7) promoters = sum(v for k, v in counter.items() if k > 8) p_detractors = (detractors / n) * 100 p_promoters = (promoters / n) * 100 nps = p_promoters - p_detractors for k in range(11): print(f"{k}: {counter.get(k, 0)}") print("\n") print(f"n= {n}") print(f"n_detractors= {detractors} ({p_detractors}%)") print(f"n_promoters= {promoters} ({p_promoters}%)") print(f"NPS= {nps}") return 0
def database_edit(ctx: Context, database: Callable[..., Database], table: str, _id: str | int, file: TextIO, submission_file: BytesIO | None, submission_thumbnail: BytesIO | None): """ Edit entries and submission files manually using a JSON file. Submission files/thumbnails can be added using the respective options. The JSON fields must match the column names of the selected table. For a list of columns for each table, please see the README at {blue}https://pypi.org/project/{prog_name}/{version}{reset}. If the {yellow}--submission-file{reset} and/or {yellow}--submission-thumbnail{reset} options are used, the {yellow}FILE{reset} argument can be omitted. """ db: Database = database() db_table: Table = get_table(db, table) data: dict = load(file) file.close() if not data and table.lower() != db.submissions.name.lower(): raise BadParameter(f"FILE cannot be empty for {table} table.", ctx, get_param(ctx, "file")) elif not isinstance(data, dict): raise BadParameter(f"Data must be in JSON object format.", ctx, get_param(ctx, "file")) add_history(db, ctx, table=table, id=_id, file=file.name, submission_file=submission_file.name if submission_file else None, submission_thumbnail=submission_thumbnail.name if submission_thumbnail else None) if (entry := db_table[_id]) is None: raise BadParameter(f"No entry with ID {_id} in {table}.", ctx, get_param(ctx, "_id"))
def display(input: TextIO) -> None: import seaborn as sns # Load view JSON. input_data: dict = json.load(input) input.close() loader = PackageLoader('mockdown.display', 'templates') env = Environment(loader=loader, autoescape=select_autoescape(['html', 'xml'])) # Kind of a hack. Using importlib.resources is probably more Pythonic but... kiwi_js_src = loader.get_source(env, 'js/flightlessbird.all.js')[0] input_meta = input_data.get('meta', {}) scrape_meta = input_data.get('scrape', {}) # Utility for convenience. # todo: move elsewhere def get_meta(*keys: str, subject=input_meta, default=None): if len(keys) == 0: return input_meta k, ks = keys[0], keys[1:] result = input_meta.get(k, default) if not ks: return result else: return get_meta(*ks, subject=result, default=default) examples = input_data['examples'] observed_bounds = { 'min_width': min(e['rect'][2] - e['rect'][0] for e in examples), 'max_width': max(e['rect'][2] - e['rect'][0] for e in examples), 'min_height': min(e['rect'][3] - e['rect'][1] for e in examples), 'max_height': max(e['rect'][3] - e['rect'][1] for e in examples), } context = { 'examples': examples, 'origin': get_meta('scrape', 'origin', default=None), 'colors': sns.color_palette('bright', len(examples)).as_hex(), **observed_bounds } template = env.get_template('default.html.jinja2') html = template.render(**context) # Write the result to a temporary file, and open it in the user's web browser. tmp = tempfile.NamedTemporaryFile(mode='w', prefix='fnord-', suffix='.html', delete=False) tmp.write(html) tmp.flush() tmp.close() webbrowser.open(f"file://{tmp.name}")
def read_raw_data(fd: TextIO, skip=1, splitter=re.compile("\t"), idx_pwd=0, idx_rank=1, idx_num=2): lst = [] for _ in range(skip): fd.readline() for line in fd: line = line.strip("\r\n") items = splitter.split(line) pwd = items[idx_pwd] rank = float(items[idx_rank]) num = int(items[idx_num]) lst.append((pwd, rank, num)) fd.close() lst = sorted(lst, key=lambda x: x[1]) num_list = [n for _, _, n in lst] total = sum(num_list) top_25 = floor(0.25 * total) bottom_25 = ceil(0.75 * total) easiest_items = [p for p, _, _ in lst[:top_25]] hardest_items = [p for p, _, _ in lst[bottom_25:]] return set(easiest_items), set(hardest_items), total
def __init__(self, json_file: TextIO, close_fd: bool, use_rate: bool = True): data = json.load(json_file) y_list = data["y_list"] total = data["total"] if use_rate: y_list = [cracked / total * 100 for cracked in y_list] if close_fd: json_file.close() elif json_file.seekable(): json_file.seek(0) self.x_list = data["x_list"] self.y_list = y_list self.color = data['color'] self.marker = data['marker'] self.marker_size = data['marker_size'] self.mark_every = data["mark_every"] self.line_width = data['line_width'] if type(data['line_style']) is str: self.line_style = data['line_style'] else: self.line_style = (data['line_style'][0], tuple(list(data['line_style'][1]))) self.label = data['label'] self.text = data['label'] self.show_text = data['show_text'] self.text_x = data['text_x'] self.text_y = data['text_y'] self.text_fontsize = data['text_fontsize'] self.text_color = data['text_color']
def read_nn(fd_csv: TextIO): pwd_dict = {} for line in fd_csv: lst = line.strip("\r\n").split('\t') pwd, prob, guess_number = lst[0:3] pwd_dict[pwd] = (float(prob), float(guess_number)) fd_csv.close() return pwd_dict
def saver(itr: Generator[Tuple[str, float, int, int, int, float]], save2: TextIO): for pwd, ml2p, num, rank, cracked, cracked_rate in itr: save2.write( f"{pwd}\t{ml2p:.8f}\t{num}\t{rank}\t{cracked}\t{cracked_rate:.2f}\n" ) save2.close() pass
def redirect_stream(input_stream: TextIO, output_stream: TextIO) -> None: while True: line = input_stream.readline() if line == "": input_stream.close() return output_stream.write(line)
def from_files(cls, schema_basepath: str, source_schema: str, target_schema: str, output_file: TextIO) -> None: source_schema_instance: Optional[Schema] = Schema.load(source_schema, base_path=schema_basepath) target_schema_instance: Optional[Schema] = Schema.load(target_schema, source_schema=source_schema_instance, base_path=schema_basepath) assert target_schema_instance is not None export: "ExportLinkages" = cls(target_schema_instance, output_file) export() output_file.close()
def _read_file(input_file: TextIO) -> str: is_stdin = input_file is sys.stdin if is_stdin: LOGGER.info("↓↓↓ Write your source code here. End with CTR^D ↓↓↓") content = input_file.read() if not is_stdin: input_file.close() return content
def freq(infile: TextIO, outfile: TextIO) -> None: count = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] #list with 10 integers # above can also use count = [0] * 10 for line in infile: count[int(line)] += 1 infile.close() for i in range(len(count)): outfile.write(f'{i} {count[i]}\n') outfile.close()
def try_to_close_output_file(self, output_file: TextIO): if output_file: try: output_file.close() logging.debug(f"Closed file {self.output_file}.") except Exception as e: logging.error( f"Error when trying to close {self.output_file}: {e}") sys.exit(EXIT_PROCESSING_ERROR)
def _read_file(input_file: TextIO) -> str: if input_file is sys.stdin: LOGGER.debug('Write your source code here. End with CTR^D') content = input_file.read() else: content = input_file.read() input_file.close() return content