def transitions_filter(file: TextIO): file.seek(0) started = False line_reader = SimpleLineGenerator(file) try: for line in line_reader(): line = line.strip() if line == '' or line[0] == "#": continue if line.lower() == "transitions": if started: raise RuntimeError( "Detected more than one 'TRANSITIONS' section. Aborting." ) else: started = True continue if line.lower() == "lambdas": for skipped_line in line_reader(): if skipped_line.strip().lower() == "transitions": if started: raise RuntimeError( "Detected more than one 'TRANSITIONS' section. Aborting." ) else: line_reader.hold_on() break if started and line.lower() != "lambdas": yield line.replace(",", ' ').split() finally: if not started: raise RuntimeError("Can not find 'TRANSITIONS' section. Aborting.")
def _new_game(self, game_id: str, infile: typing.TextIO): """Consumes lines describing game metadata from the event file. Args: game_id: Identifier for the game. infile: Open buffer reading the event file. Returns: None. """ self.current_game = {"id": game_id} while True: prev_loc = infile.tell() line = infile.readline() if not line: raise Exception("Encountered EOF while parsing new game info") fields = line.strip().split(",") if fields[0] not in ("version", "info"): infile.seek(prev_loc) return if fields[0] == "info": field, value = fields[1:] self.current_game[field] = value if field == "visteam": self.v_roster = data.get_roster(self.year, value) elif field == "hometeam": self.h_roster = data.get_roster(self.year, value)
def load(filestream: TextIO) -> Dict[str, Any]: """Load and parse a YAML-formatted file. :param filename: The YAML file to load. :raises SnapcraftError: if loading didn't succeed. :raises LegacyFallback: if the project's base is not core22. """ try: data = yaml.safe_load(filestream) build_base = utils.get_effective_base( base=data.get("base"), build_base=data.get("build-base"), project_type=data.get("type"), name=data.get("name"), ) if build_base is None: raise errors.LegacyFallback("no base defined") if build_base != "core22": raise errors.LegacyFallback("base is not core22") except yaml.error.YAMLError as err: raise errors.SnapcraftError( f"snapcraft.yaml parsing error: {err!s}") from err filestream.seek(0) try: return yaml.load(filestream, Loader=_SafeLoader) except yaml.error.YAMLError as err: raise errors.SnapcraftError( f"snapcraft.yaml parsing error: {err!s}") from err
def _read_game(self, infile: typing.TextIO): """Read events and generate data until a new game is declared in the file. Args: infile: An open buffer reading the event file. Yields: A row of tabular data for every play in the game. """ while True: prev_loc = infile.tell() line = infile.readline() if not line: return fields = line.strip().split(",") if fields[0] == "id": infile.seek(prev_loc) return if fields[0] in ["start", "sub"]: pid: str = fields[1] pos: int = int(fields[5]) lineup = self.h_lineup if int(fields[3]) else self.v_lineup lineup[pos - 1] = pid continue if fields[0] == "play": self._process_play(*fields[1:]) yield self._current_event self._current_event = None
def get_file_size(file: TextIO) -> int: """Get size of file in bytes without altering file""" starting_pos = file.tell() file.seek(0, os.SEEK_END) file_size = file.tell() file.seek(0, starting_pos) return file_size
def get_first_chrom_name(fp: TextIO) -> str: # make sure file pointer is at beginning of file fp.seek(0) header = fp.readline().rstrip() if not header.startswith(">"): raise ValueError(f"Expected fasta file, but it did not start with '>'") return header[1:].split()[0]
def scan_draws(fp: TextIO, config_dict: Dict, lineno: int) -> int: """ Parse draws, check elements per draw, save num draws to config_dict. """ draws_found = 0 num_cols = len(config_dict['column_names']) cur_pos = fp.tell() line = fp.readline().strip() first_draw = None while len(line) > 0 and not line.startswith('#'): lineno += 1 draws_found += 1 data = line.split(',') if len(data) != num_cols: raise ValueError( 'line {}: bad draw, expecting {} items, found {}'.format( lineno, num_cols, len(line.split(',')))) if first_draw is None: first_draw = np.array(data, dtype=np.float64) cur_pos = fp.tell() line = fp.readline().strip() config_dict['draws'] = draws_found config_dict['first_draw'] = first_draw fp.seek(cur_pos) return lineno
def _read_energy(f_object: TextIO): """ Read the energy value from a force file. Parameters ---------- f_object : TextIO File object from which to read the values Returns ------- Energy value for that configuration : float """ energy = None pattern = "Total FORCE_EVAL" for line in f_object: if re.search(pattern, line): energy = float(line.split()[-1]) if energy is None: print("No energy value found - Are you sure the input file was correct? Check the optimization.out file " "for more details.") sys.exit(1) else: f_object.seek(0) # return to the start of the file. return energy
def _read_forces(f_object: TextIO, n_atoms: int): """ Read in the forces from the text file. Parameters ---------- f_object : TextIO file object from which to read data n_atoms : int Number of atoms in the file. Returns ------- mean force value as a float : float """ force = 0.0 pattern_1 = "# Atom Kind Element X Y Z" pattern_2 = "SUM OF ATOMIC FORCES" for i, line, in enumerate(f_object): if re.search(pattern_1, line): start = i + 1 elif re.search(pattern_2, line): stop = i - 1 f_object.seek(0) for i, line in enumerate(f_object): if i < start: continue elif start <= i <= stop: force += np.linalg.norm(np.array(line.split())[3:].astype(float)) else: break f_object.seek(0) return force/n_atoms
def strip_vars(f: typing.TextIO) -> list: tmp_lst = [] i = 0 count_var_regex = r"\bcount\b" count_var_regex_sub = COUNT_REPLACEMENT_WORD f.seek(0) try: for line in f.readlines(): # Changes var.count, but not var.count_of_consul, for example repl_count_var = re.sub(count_var_regex, count_var_regex_sub, line, 0) tmp_lst.append(repl_count_var) if line.startswith(tuple(ignore)): # This assumes your .tf files have a contiguous block of variables tmp_lst.pop() break for line in "".join(tmp_lst).splitlines(): if line.startswith("variable"): break while not line.startswith("variable"): i += 1 break del tmp_lst[:i] # Remove trailing newline if it got pulled in if "".join(tmp_lst[-1:]).isspace(): del tmp_lst[-1:] # And add a newline to the head for any existing entries if not "".join(tmp_lst[0]).isspace(): tmp_lst.insert(0, "\n") except IndexError: if len(sys.argv) > 1: print("INFO: No variables found in " + f.name) return tmp_lst
def _detect_dialect(cls, csv_file: TextIO, dialect: Dialect) -> Dialect: number_of_rows = 1024 start_of_file = 0 if not dialect: dialect = Sniffer().sniff(csv_file.read(number_of_rows)) csv_file.seek(start_of_file) return dialect
def get_file_size(f: typing.TextIO) -> int: """ Gets file size. This function restores the file position. """ restore = f.tell() f.seek(0, 2) ret = f.tell() f.seek(restore) return ret
def get_imports(blueprint_file: typing.TextIO) -> dict: level = 0 imports_token = None import_lines = {} blueprint_file.seek(0, 0) for t in yaml.scan(blueprint_file): if isinstance(t, (yaml.tokens.BlockMappingStartToken, yaml.tokens.BlockSequenceStartToken, yaml.tokens.FlowMappingStartToken, yaml.tokens.FlowSequenceStartToken)): level += 1 if isinstance( t, (yaml.tokens.BlockEndToken, yaml.tokens.FlowMappingEndToken, yaml.tokens.FlowSequenceEndToken)): level -= 1 if isinstance(t, yaml.tokens.ScalarToken): if level == 1 and t.value == 'imports': imports_token = t continue token_length = t.end_mark.index - t.start_mark.index if level >= 1 and imports_token and \ token_length < MAX_IMPORT_TOKEN_LENGTH: import_lines[t.value] = { START_POS: t.start_mark.index, END_POS: t.end_mark.index, } if isinstance(t, yaml.tokens.KeyToken) and imports_token: break return import_lines
def main(csv_file: TextIO, output_dir: Path) -> int: output_dir.mkdir(exist_ok=True) dialect = csv.Sniffer().sniff(csv_file.read(1024 * 4)) csv_file.seek(0) reader = csv.DictReader(csv_file, dialect=dialect) rows = list(reader) csv_file.close() def is_question(q: str) -> bool: return q not in ["Timestamp", "Email address"] questions = filter(is_question, rows[0].keys()) data = {q: get_data(rows, q) for q in questions} max_n = max(max(v[1]) for v in data.values()) print(max_n) y_ticks = np.arange(0, max_n + 2, step=2) for question, counter in data.items(): filename = safe_filename(question) + ".png" filename = output_dir / filename create_graph(counter, question, filename, y_ticks=y_ticks) return 0
def get_mzXMLs_from_pep_xml(pepxml_file: typing.TextIO): REC = re.compile(' base_name="(.+?)"') currpos = pepxml_file.tell() t = pepxml_file.read() pepxml_file.seek(currpos) paths = map(pathlib.Path, REC.findall(t)) return [path.with_suffix(".mzXML") for path in paths if path.is_absolute()]
def one_file(fp: TextIO) -> None: lines = list(fp) new = lines[:] shebang_header = False if new[0].startswith("#!"): shebang_header = True new.pop(0) if not new[0].strip(): new.pop(0) while new and new[0][0] == "#": new.pop(0) while new and new[0].strip() == "": new.pop(0) new.insert(0, LICENSE_NOTICE) if shebang_header: new.insert(0, SHEBANG_HEADER) data = "".join(new) fp.seek(0) fp.write(data) fp.truncate()
def __init__(self, file: TextIO): self.text: str = file.read() self.current_index: int = 0 self.current_token: Optional[str] = None self.next_index: Optional[int] = None self.next_token: Optional[str] = None file.seek(0)
def from_file(cls, fp: TextIO, skip_line=2): id = fp.readline() # print(id) if id is None: raise RuntimeError('No more line exist in {}'.format(fp)) for i in range(skip_line): fp.readline() size = eval(fp.readline()) data = [] for i in range(size): now_position = fp.tell() line = fp.readline() try: x, y = map(float, line.split()) data.append([x, y]) except Exception as e: logger.warning( f"Except point number in {fp} is {size} but only get {i} points" ) # 拟合数据中的数据长度不等于self,_size fp.seek(now_position) # 回到上一行 size = i # 更新点的数量 break return SinglePoint(size, data)
def repair_depfile(depfile: TextIO, include_dirs: List[Path]) -> None: changes_made = False out = "" for line in depfile.readlines(): if ":" in line: colon_pos = line.rfind(":") out += line[: colon_pos + 1] line = line[colon_pos + 1 :] line = line.strip() if line.endswith("\\"): end = " \\" line = line[:-1].strip() else: end = "" path = Path(line) if not path.is_absolute(): changes_made = True path = resolve_include(path, include_dirs) out += f" {path}{end}\n" # If any paths were changed, rewrite the entire file if changes_made: depfile.seek(0) depfile.write(out) depfile.truncate()
def from_mypy(self, file: TextIO) -> List[Issue]: buff = None items = [] for line in file: line = line.strip() if RE_MYPY_LINE.match(line): if buff: items.append(self.__parse_mypy_issue(buff)) buff = [] if buff is not None: buff.append(line) if buff is None: # mypy can return an error without line/column values file.seek(0) for line in file: m = RE_MYPY_LINE_WO_COORDINATES.match(line.strip()) if not m: continue items.append( self.Issue( path=m.group(1).strip(), description=m.group(2).strip(), rule=self._mypy_rules["[unknown]"], )) self.logger.info( f"detected an error without coordinates: {line}") items.append(self.__parse_mypy_issue(buff)) return [i for i in items if i]
def check_stream( input_stream: TextIO, show_diff: Union[bool, TextIO] = False, extension: Optional[str] = None, config: Config = DEFAULT_CONFIG, file_path: Optional[Path] = None, disregard_skip: bool = False, **config_kwargs, ) -> bool: """Checks any imports within the provided code stream, returning `False` if any unsorted or incorrectly imports are found or `True` if no problems are identified. - **input_stream**: The stream of code with imports that need to be sorted. - **show_diff**: If `True` the changes that need to be done will be printed to stdout, if a TextIO stream is provided results will be written to it, otherwise no diff will be computed. - **extension**: The file extension that contains imports. Defaults to filename extension or py. - **config**: The config object to use when sorting imports. - **file_path**: The disk location where the code string was pulled from. - **disregard_skip**: set to `True` if you want to ignore a skip set in config for this file. - ****config_kwargs**: Any config modifications. """ config = _config(path=file_path, config=config, **config_kwargs) changed: bool = sort_stream( input_stream=input_stream, output_stream=Empty, extension=extension, config=config, file_path=file_path, disregard_skip=disregard_skip, ) printer = create_terminal_printer(color=config.color_output) if not changed: if config.verbose and not config.only_modified: printer.success(f"{file_path or ''} Everything Looks Good!") return True else: printer.error(f"{file_path or ''} Imports are incorrectly sorted and/or formatted.") if show_diff: output_stream = StringIO() input_stream.seek(0) file_contents = input_stream.read() sort_stream( input_stream=StringIO(file_contents), output_stream=output_stream, extension=extension, config=config, file_path=file_path, disregard_skip=disregard_skip, ) output_stream.seek(0) show_unified_diff( file_input=file_contents, file_output=output_stream.read(), file_path=file_path, output=None if show_diff is True else cast(TextIO, show_diff), color_output=config.color_output, ) return False
def parse_conllu_plus_fields( in_file: T.TextIO, metadata_parsers: T.Optional[T.Dict[str, _MetadataParserType]] = None ) -> T.Optional[T.Sequence[str]]: pos = in_file.tell() # Get first line try: first_sentence = next(parse_sentences(in_file)) first_line = first_sentence.split("\n")[0] except StopIteration: first_line = "" # parse_sentences moves to file cursor, so reset it here in_file.seek(pos) if not first_line.startswith("#"): return None tokenlist = parse_token_and_metadata(first_line, metadata_parsers=metadata_parsers) metadata = tokenlist.metadata fields = None if "global.columns" in metadata and metadata["global.columns"]: fields = [ value.lower() for value in metadata["global.columns"].split(" ") ] return fields
def _get_metadata(fp: TextIO) -> Tuple[int, Dict[str, str]]: """ Load metadata from a fluke 985 tab-delimited data file. Parameters ---------- fp : file-like object The data file. Returns ------- last_md_line : int The index of the blank line between metadata and the table. metadata : dict A dictionary of metadata, including "Model Number" and others specified in the header. """ metadata = {} last_md_line = 0 fp.seek(0) for line_number in itertools.count(start=1): line = fp.readline().strip() if ':' not in line: last_md_line = line_number break key, value = line.split(':', 1) metadata[key.strip()] = value.strip() return last_md_line, metadata
def scan_config(fd: TextIO, config_dict: Dict[str, Any], lineno: int) -> int: """ Scan initial stan_csv file comments lines and save non-default configuration information to config_dict. """ cur_pos = fd.tell() line = fd.readline().strip() while len(line) > 0 and line.startswith('#'): lineno += 1 if line.endswith('(Default)'): line = line.replace('(Default)', '') line = line.lstrip(' #\t') key_val = line.split('=') if len(key_val) == 2: if key_val[0].strip() == 'file' and not key_val[1].endswith('csv'): config_dict['data_file'] = key_val[1].strip() elif key_val[0].strip() != 'file': raw_val = key_val[1].strip() val: Union[int, float, str] try: val = int(raw_val) except ValueError: try: val = float(raw_val) except ValueError: val = raw_val config_dict[key_val[0].strip()] = val cur_pos = fd.tell() line = fd.readline().strip() fd.seek(cur_pos) return lineno
def scan_sampling_iters(fd: TextIO, config_dict: Dict[str, Any], lineno: int) -> int: """ Parse sampling iteration, save number of iterations to config_dict. """ draws_found = 0 num_cols = len(config_dict['column_names']) cur_pos = fd.tell() line = fd.readline().strip() while len(line) > 0 and not line.startswith('#'): lineno += 1 draws_found += 1 data = line.split(',') if len(data) != num_cols: raise ValueError( 'line {}: bad draw, expecting {} items, found {}\n'.format( lineno, num_cols, len(line.split(','))) + 'This error could be caused by running out of disk space.\n' 'Try clearing up TEMP or setting output_dir to a path' ' on another drive.', ) cur_pos = fd.tell() line = fd.readline().strip() config_dict['draws_sampling'] = draws_found fd.seek(cur_pos) return lineno
def load(fp: t.TextIO) -> t.Union[NNF, And[Or[Var]]]: """Load a sentence from an open file. The format is automatically detected. """ for line in fp: if line.startswith('c'): continue if line.startswith('p '): problem = line.split() if len(line) < 2: raise DecodeError("Malformed problem line") fmt = problem[1] if 'sat' in fmt or 'SAT' in fmt: # problem[2] contains the number of variables # but that's currently not explicitly represented return _load_sat(fp) elif 'cnf' in fmt or 'CNF' in fmt: # problem[2] has the number of variables # problem[3] has the number of clauses return _load_cnf(fp) else: raise DecodeError("Unknown format '{}'".format(fmt)) elif line.startswith('nnf '): # Might be a DSHARP output file from nnf import dsharp fp.seek(0) return dsharp.load(fp) else: raise DecodeError( "Couldn't find a problem line before an unknown kind of line") else: raise DecodeError( "Couldn't find a problem line before the end of the file")
def countCharacters(file: typing.TextIO, background: str, delta_x: int, delta_y: int, wrap: DIRECTION) -> int: count = 0 x_position = 0 lines = iter(file) line = next(lines).rstrip() while True: delta_lines = delta_y while delta_lines > 0: try: line = next(lines).rstrip() delta_lines -= 1 except StopIteration: if wrap != 'y': break file.seek(0) if delta_lines > 0: break x_position += delta_x if x_position >= len(line): if wrap != 'x': break x_position %= len(line) if line[x_position] != background: count += 1 return count
def __init__(self, json_file: TextIO, close_fd: bool, use_rate: bool = True): data = json.load(json_file) y_list = data["y_list"] total = data["total"] if use_rate: y_list = [cracked / total * 100 for cracked in y_list] if close_fd: json_file.close() elif json_file.seekable(): json_file.seek(0) self.x_list = data["x_list"] self.y_list = y_list self.color = data['color'] self.marker = data['marker'] self.marker_size = data['marker_size'] self.mark_every = data["mark_every"] self.line_width = data['line_width'] if type(data['line_style']) is str: self.line_style = data['line_style'] else: self.line_style = (data['line_style'][0], tuple(list(data['line_style'][1]))) self.label = data['label'] self.text = data['label'] self.show_text = data['show_text'] self.text_x = data['text_x'] self.text_y = data['text_y'] self.text_fontsize = data['text_fontsize'] self.text_color = data['text_color']
def count_lines(f_: TextIO, chunksize=2 ** 12) -> int: count = 0 f_.seek(0) chunk = f_.read(chunksize) while chunk: count += chunk.count(NEWLINE) chunk = f_.read(chunksize) return count
def is_black_alpha_only(data: TextIO) -> bool: """Check if an image has only black pixels (with alpha)""" result = False with Image.open(data) as image: if image.mode == "RGBA": result = not any(p[:3] != (0, 0, 0) for p in list(image.getdata())) data.seek(0) return result
def dump_conditions(file: TextIO) -> None: """Dump docs for all the condition flags, results and metaconditions.""" LOGGER.info('Dumping conditions...') # Delete existing data, after the marker. file.seek(0, io.SEEK_SET) prelude = [] for line in file: if DOC_MARKER in line: break prelude.append(line) file.seek(0, io.SEEK_SET) file.truncate(0) if not prelude: # No marker, blank the whole thing. LOGGER.warning('No intro text before marker!') for line in prelude: file.write(line) file.write(DOC_MARKER + '\n\n') file.write(DOC_META_COND) ALL_META.sort(key=lambda i: i[1]) # Sort by priority for flag_key, priority, func in ALL_META: file.write('#### `{}` ({}):\n\n'.format(flag_key, priority)) dump_func_docs(file, func) file.write('\n') for lookup, name in [ (ALL_FLAGS, 'Flags'), (ALL_RESULTS, 'Results'), ]: print('<!------->', file=file) print('# ' + name, file=file) print('<!------->', file=file) lookup_grouped = defaultdict(list) # type: Dict[str, List[Tuple[str, Tuple[str, ...], Callable]]] for flag_key, aliases, func in lookup: group = getattr(func, 'group', 'ERROR') if group is None: group = '00special' lookup_grouped[group].append((flag_key, aliases, func)) # Collapse 1-large groups into Ungrouped. for group in list(lookup_grouped): if len(lookup_grouped[group]) < 2: lookup_grouped[''].extend(lookup_grouped[group]) del lookup_grouped[group] if not lookup_grouped['']: del lookup_grouped[''] for header_ind, (group, funcs) in enumerate(sorted(lookup_grouped.items())): if group == '': group = 'Ungrouped Conditions' if header_ind: # Not before the first one... print('---------\n', file=file) if group == '00special': print(DOC_SPECIAL_GROUP, file=file) else: print('### ' + group + '\n', file=file) LOGGER.info('Doing {} group...', group) for flag_key, aliases, func in funcs: print('#### `{}`:\n'.format(flag_key), file=file) if aliases: print('**Aliases:** `' + '`, `'.join(aliases) + '`' + ' \n', file=file) dump_func_docs(file, func) file.write('\n')