def execute(self, env: dict, stdin: IOBase, stdout: IOBase, stderr: IOBase) -> None: if not self.args: stdout.write(self._count(stdin)) return for arg in self.args: with open(arg, "r") as f: stdout.write(self._count(f))
def dump(obj: Any, fp: io.IOBase, cls=None, **kwargs): if cls is None: cls = JsonlEncoder text = cls(**kwargs).encode(obj) if(isinstance(fp, (io.RawIOBase, io.BufferedIOBase))): fp.write(text.encode()) else: fp.write(text)
def write_binary128(value: bytes, writer: io.IOBase) -> None: """ Writes a binary128 floating point value to the writer. It will be encoded using IEEE 754-2008 in big endian value. """ if len(value) != 16: raise ValueError('Invalid binary128.') writer.write(value)
def dumpf(self, obj, fp: IOBase, options: dict): 'dump a obj into the file-like object.' assert fp.writable() if isinstance(fp, _TextIOBase): fp.write(self.dumps(obj, options)) else: fp.write(self.dumpb(obj, options))
def dump(self, f: io.IOBase): """ Dump data to json lines format. Each json object is formatted as `{key: [value1, value2, ...]}`. Args: f (io.IOBase): IO handler. """ for k, ss in self: obj = {k: list(ss)} f.write(json.dumps(obj) + '\n')
def retrieve(self, key: str, stream: IOBase): try: blob_client = self.client.get_blob_client(key) download_stream = blob_client.download_blob() stream.write(download_stream.readall()) except Exception as ex: logging.error( "ERROR: Error occurred while try to perform retrieve operation for key %s", key) raise ex
def execute(self, env: dict, stdin: IOBase, stdout: IOBase, stderr: IOBase) -> None: if not self.args: for line in stdin: stdout.write(line) return for arg in self.args: with open(arg, "r") as f: for line in f: stdout.write(line)
def generate_code(atcoder_client: AtCoderClient, problem_url: str, config: Config, output_file: IOBase): problem = get_problem_from_url(problem_url) template_code_path = config.code_style_config.template_file lang = config.code_style_config.lang def emit_error(text): logging.error(with_color(text, Fore.RED)) def emit_warning(text): logging.warning(text) def emit_info(text): logging.info(text) emit_info('{} is used for template'.format(template_code_path)) # Fetch problem data from the statement try: content = atcoder_client.download_problem_content(problem) except InputFormatDetectionError as e: emit_error("Failed to download input format.") raise e except SampleDetectionError as e: emit_error("Failed to download samples.") raise e try: prediction_result = predict_format(content) emit_info( with_color("Format prediction succeeded", Fore.LIGHTGREEN_EX)) except (NoPredictionResultError, MultiplePredictionResultsError) as e: prediction_result = FormatPredictionResult.empty_result() if isinstance(e, NoPredictionResultError): msg = "No prediction -- Failed to understand the input format" else: msg = "Too many prediction -- Failed to understand the input format" emit_warning(with_color(msg, Fore.LIGHTRED_EX)) constants = predict_constants(content.original_html) code_generator = config.code_style_config.code_generator with open(template_code_path, "r") as f: template = f.read() output_splitter() output_file.write(code_generator( CodeGenArgs( template, prediction_result.format, constants, config.code_style_config )))
def _write_xml_output_to_file(self, file_io: io.IOBase): """Write the dump from self._dump() to file_io. Args: file_io: File to write to. """ output_str = self._dump() ExtendedCommonRoadFileWriter.check_validity_of_commonroad_file( output_str) file_io.write(output_str)
def write_cooccurrences(cooccurrences: List[ProductsCoOccurrence], output_file: io.IOBase): ''' write transactions to file in the format: <item set size (N)>, <co-occurrence frequency>, <item 1 id >, <item 2 id>, …. <item N id> ''' for cooccurrence in cooccurrences: output_file.write('{:d}, {:d}, {:s}\n'.format( len(cooccurrence.products_set), cooccurrence.co_occurrence_frequency, ', '.join(cooccurrence.products_set)))
def report( self, fout: IOBase, ct_list: List["CellType"], space: int = 4, ) -> None: """ Write putative cell type reports to fout. """ for ct in ct_list: fout.write(" " * space + str(ct) + "\n") if ct.subtypes is not None: self.report(fout, ct.subtypes, space + 4)
def generate_code(atcoder_client: AtCoderClient, problem_url: str, config: Config, output_file: IOBase): problem = get_problem_from_url(problem_url) template_code_path = config.code_style_config.template_file lang = config.code_style_config.lang def emit_error(text): logging.error(with_color(text, Fore.RED)) def emit_warning(text): logging.warning(text) def emit_info(text): logging.info(text) emit_info('{} is used for template'.format(template_code_path)) # Fetch problem data from the statement try: content = atcoder_client.download_problem_content(problem) except InputFormatDetectionError as e: emit_error("Failed to download input format.") raise e except SampleDetectionError as e: emit_error("Failed to download samples.") raise e try: prediction_result = predict_format(content) emit_info(with_color("Format prediction succeeded", Fore.LIGHTGREEN_EX)) except (NoPredictionResultError, MultiplePredictionResultsError) as e: prediction_result = FormatPredictionResult.empty_result() if isinstance(e, NoPredictionResultError): msg = "No prediction -- Failed to understand the input format" else: msg = "Too many prediction -- Failed to understand the input format" emit_warning(with_color(msg, Fore.LIGHTRED_EX)) constants = predict_constants(content.original_html) code_generator = config.code_style_config.code_generator with open(template_code_path, "r") as f: template = f.read() output_splitter() output_file.write( code_generator( CodeGenArgs(template, prediction_result.format, constants, config.code_style_config)))
def _log_fit_params(self, f: IOBase): f.write("Fit parameters\n") f.write("Epochs: {}\n".format(self.epochs)) f.write("Batch size: {}\n".format(self.batch_size)) f.write( "_________________________________________________________________\n\n" )
def _log_directories(self, f: IOBase): f.write("Dataset directory: {}\n".format(self.dataset_dir)) f.write("dtype: {}\n".format(self.dtype)) f.write("Checkpoints directory: {}\n".format(self.checkpoints_dir)) f.write( "_________________________________________________________________\n\n" )
def generate(self, result: scanner.ScannerResult, output: io.IOBase) -> None: for decl in filter( lambda decl: decl.type.args is not None and any( map( lambda param: (not isinstance(param, EllipsisParam) and utils. is_function_pointer_type(param.type)), decl.type.args.params, )), result.declarations, ): output.write(self._generateTypeDefForDecl(decl)) for defin in filter( lambda defin: defin.decl.type.args is not None and any( map( lambda param: (not isinstance(param, EllipsisParam) and utils. is_function_pointer_type(param.type)), defin.decl.type.args.params, )), result.definitions, ): output.write(self._generateTypeDefForDecl(defin.decl)) for decl in result.declarations: output.write(self._generateFakeForDecl(decl)) for definition in result.definitions: output.write(self._generateBypassForFuncDef(definition)) output.write(self._generateFakeForDecl(definition.decl))
def download(url: str, stream: io.IOBase, timeout: int, verify: bool): """ Downloads the content of URL into stream. This method only supports HTTP and HTTPS URLs. The implementation is safe to use for large contents. :param url: URL to download. :param stream: stream to write content (e.g. file or I/O buffer). :param timeout: timeout until server sends data (not the overall download time). :param verify: verify server's SSL certificate. """ with requests.get(url, timeout=timeout, verify=verify, stream=True) as r: r.raise_for_status() for chunk in r.iter_content(chunk_size=1024 * 1024): stream.write(chunk)
def write_int(value: int, size: int, signed: bool, writer: io.IOBase): """ Writes an integer into the writer. It is always encoded as a big endian value. Parameters: - `value`: The value to be written; - `size`: The size of the value in bytes; - `signed`: A flag that determines if the encoding is signed or not; - `writer`: The writer; It may raise `OverflowError` if the `value` cannot be represented using the specfied size. """ writer.write(value.to_bytes(size, byteorder='big', signed=signed))
def _serialize(self, out_stream: io.IOBase, data: list): if not data: return encoding = "utf-8" delimeter = '\t' out_stream.write("value{d}signer{d}sign\n".format(d=delimeter).encode(encoding)) for row in data: out_stream.write("{value}{d}{signer}{d}{sign}\n".format( d=delimeter, value=json.dumps(row.data), signer=str(base64.b64encode(row.signer), encoding='utf-8') if row.signer else "", sign=str(base64.b64encode(row.sign), encoding='utf-8') if row.sign else "" ).encode(encoding))
def dump(f: io.IOBase, xss: List[List[Union[bytes, int, float]]]) -> None: buffer = io.BytesIO() for xs in xss: # write max index assert _sizeof[_uint16] == buffer.write( struct.pack(_uint16, len(xs) - 1)) # write types for x in xs: if isinstance(x, bytes): x = _BSV_CHAR elif isinstance(x, int): x = _BSV_INT elif isinstance(x, float): x = _BSV_FLOAT else: assert False assert _sizeof[_uint8] == buffer.write(struct.pack(_uint8, x)) # write sizes for x in xs: if isinstance(x, bytes): x = len(x) elif isinstance(x, int): x = _sizeof[_bsv_int] elif isinstance(x, float): x = _sizeof[_bsv_float] else: assert False assert _sizeof[_uint16] == buffer.write(struct.pack(_uint16, x)) # write vals for x in xs: if isinstance(x, bytes): pass elif isinstance(x, int): x = struct.pack(_bsv_int, x) elif isinstance(x, float): x = struct.pack(_bsv_float, x) else: assert False assert len(x) == buffer.write(x) assert 1 == buffer.write(b'\0') assert _sizeof[_int32] == f.write( struct.pack(_int32, len(buffer.getvalue()))) assert len( buffer.getvalue() ) < _buffer_size, f'you cant dump more than {_buffer_size} bytes at a time' assert len(buffer.getvalue()) == f.write(buffer.getvalue())
def write_chunk(bs: io.IOBase, magic: bytes, body: bytes): body_size = len(body) body_size_padding = get_padding_size(body_size) body_size += body_size_padding bs.write(int.to_bytes(body_size, length=4, byteorder='little')) bs.write(magic) bs.write(body) for _ in range(body_size_padding): bs.write(b' ')
def _write(f: io.IOBase, param: str): try: write_str = param.encode() written_bytes_len = f.write(write_str) except OSError: print("cloud not seek file.") return os.EX_IOERR else: print(f"wrote {written_bytes_len} bytes.")
def streamcopy(from_: IOBase, to_: IOBase, size=-1, chunk_size=163840): '''Copies content from one buffer to the other,chunk by chunk NOTE: Nor from_ or to_ has to be IOBase objects,it's doable as long as they have read() / write() calls Args: from_ (IOBase): Stream to copy from to_ (IOBase): Stream to copy to size (int, optional): Length to be copied. Defaults to -1. chunk_size (int, optional): Size of chunk. Defaults to 163840. Returns: int : copied length ''' if not size: return 0 size, copied = int(size), 0 if size < 0: # read until EOF def copychunk(): chunk = from_.read(chunk_size) if not chunk: return 0 to_.write(chunk) return len(chunk) while (True): copied_ = copychunk() if not copied_: break copied += copied_ else: # read `size` of bytes for offset in range(0, size, chunk_size): remaining = size - offset chunk = from_.read( remaining if remaining < chunk_size else chunk_size) if not chunk: break copied += len(chunk) to_.write(chunk) return copied
def pretty_print( xml: Union[str, List[Union[Element, str]], Element], file: IOBase = sys.stdout, ): """Prints beautiful XML-Code This function gets a string containing the xml, an object of List[lxml.etree.Element] or directly a lxml element. Print it with good readable format. Arguments: xml (str, List[lxml.etree.Element] or lxml.etree.Element): xml as string, List[lxml.etree.Element] or directly a lxml element. file: A IOBase type. Can be a File, StringIO, ... """ if not isinstance(file, IOBase): raise TypeError( f"Type needs to be from IOBase, not {type(file)}.") from None if isinstance(xml, list): for item in xml: if isxmlelement(item): file.write( xmltostring( item, pretty_print=True).decode(sys.getdefaultencoding() + "\n")) else: file.write(item + "\n") elif isxmlelement(xml): file.write( xmltostring(xml, pretty_print=True).decode(sys.getdefaultencoding() + "\n")) elif isinstance(xml, str): tree = secET.fromstring(xml) file.write( xmltostring(tree, pretty_print=True).decode(sys.getdefaultencoding() + "\n")) else: raise InvalidArgumentType(function=pretty_print.__name__, argument="xml", arg_type="")
def _log_optimizer(self, f: IOBase): opt_name = self.optimizer.__class__.__name__ f.writelines("Optimizer: {}\n".format(opt_name)) if opt_name == "Adam": f.writelines([ "lr: {}\n".format(K.eval(self.optimizer.lr)), "beta_1: {}\n".format(K.eval(self.optimizer.beta_1)), "beta_2: {}\n".format(K.eval(self.optimizer.beta_2)), "epsilon: {}\n".format(self.optimizer.epsilon), "decay: {}\n".format(K.eval(self.optimizer.decay)) ]) elif opt_name == "SGD": f.writelines([ "lr: {}\n".format(K.eval(self.optimizer.lr)), "momentum: {}\n".format(K.eval(self.optimizer.momentum)), "decay: {}\n".format(K.eval(self.optimizer.decay)), "initial_decay: {}\n".format(self.optimizer.initial_decay), "nesterov: {}\n".format(self.optimizer.nesterov) ]) f.write( "_________________________________________________________________\n\n" )
def _log_noise(self, f: IOBase): if isinstance(self.sigma, dict): # Attribute sigma should not be a dict unless using the FuelCell dataset assert list(self.sigma.keys()) == ['A', 'B', 'C', 'D', 'E'] f.write("Noise2Noise: {}\n".format(self.noise2noise)) f.write("Noise level: {}\n".format(self.sigma)) f.write( "_________________________________________________________________\n\n" )
def collapse(barcode_distance: int, barcode_mask: str, verbose: bool = False, inStream: io.IOBase = stdin, outStream: io.IOBase = stdout, logStream: io.IOBase = stderr): """ Collapse reads sharing the same start position, forward/reverse, and barcode into a single family record. :param input: A file or stream handle to read input data :param output: A file or stream handle to output data :param barcode_distance: The maximum number of differences allowed to be considered the same family :param barcode_mask: String mask to denote the positions within a barcode to include in the family name :param verbose: Provide verbose output while processing :param logStream: The stream to write log output to, defaults to stderr :return: None """ if verbose: logStream.write("\n") # This will be deleted by the next write inFile = AlignmentFile(inStream) outFile = AlignmentFile( outStream, "wbu" if hasattr(outStream, 'name') else "wb", template=inFile) # compress the data if it is a file families = Families( barcode_mask * 2, barcode_distance ) # reads have mate barcode concatenated so double up mask # TODO multiply distance by 2? count = 0 fcount = 0 lastFCount = 0 lastFamilyPos = 0 for family in CoordinateSortedInputFamilyIterator( inFile, families): #type: FamilyRecord outFile.write(family.toPysam()) count += len(family) fcount += 1 if verbose and lastFamilyPos != family.pos: logStream.write( "\x1b[F\x1b[2K\r{file}\tRecords processed: {count}\t" "Positions buffered: {pos}\tFamily records: {fcount}\t" "Families at last position: {lastFCount}\n".format( file=inStream.name if hasattr(inStream, 'name') else 'Streaming', count=count, pos=len(families), fcount=fcount, lastFCount=fcount - lastFCount)) lastFamilyPos = family.pos lastFCount = fcount if verbose: logStream.write("Completed.\n") inFile.close() outFile.close() outStream.close()
def pretty_print(xml, file: IOBase = sys.stdout): """Prints beautiful XML-Code This function gets a string containing the xml, an object of List[lxml.etree.Element] or directly a lxml element. Print it with good readable format. Arguments: xml (str, List[lxml.etree.Element] or lxml.etree.Element): xml as string, List[lxml.etree.Element] or directly a lxml element. file: A IOBase type. Can be a File, StringIO, ... """ if not isinstance(file, IOBase): raise TypeError( 'Type needs to be from IOBase, not {}.'.format(type(file)) ) from None if isinstance(xml, list): for item in xml: if etree.iselement(item): file.write( etree.tostring(item, pretty_print=True).decode( sys.getdefaultencoding() + '\n' ) ) else: file.write(item + '\n') elif etree.iselement(xml): file.write( etree.tostring(xml, pretty_print=True).decode( sys.getdefaultencoding() + '\n' ) ) elif isinstance(xml, str): tree = secET.fromstring(xml) file.write( etree.tostring(tree, pretty_print=True).decode( sys.getdefaultencoding() + '\n' ) )
def dump(self, obj: any, stream: IOBase): if not isinstance(obj, str): raise UnserializableTypeError('Only str type is supported ' 'in StringSerializer!') return stream.write(obj.encode('utf-8'))
def dump(self, obj: any, stream: IOBase): if not isinstance(obj, bytes): raise UnserializableTypeError('Only bytes type is supported ' 'in BinarySerializer!') return stream.write(obj)
def bench_accuracy( self, file_output: IOBase, tester: AccuracyRussianWord, image: np.ndarray, depth: np.ndarray, filename: str, ) -> bool: """ Runs all text detection accuracy benchmarks on an image. Outputs results to csv file. Parameters ---------- file_output: IO File stream to output results to. tester: AccuracyRussianWord Benchmark class to use. Initialized elsewhere to be static between image runs. image: np.ndarray The color image from the frame. depth: np.ndarray The depth image from the frame. filename: str The name of the image file. Returns ------- bool - True if test failed at some point. False if test did not crash and ran to completion. """ crash = False ## Run tests on the image ## bboxes = [] # detect_russian_word if not crash: try: bboxes = tester.accuracy_detector(color_image=image, depth_image=depth) # output BoundingBoxes to text file bbstr = [((b.__repr__()) + "\n") for b in bboxes] bbstr = "".join(bbstr) output_file_name = os.path.splitext(filename)[0] + ".txt" output_file_path = os.path.join(self.OUTPUT_RESULTS_DIR, output_file_name) if not bbstr: bbstr = None with open(output_file_path, mode="w") as text_data_output: text_data_output.write("BoundingBoxes: " + ( str(bbstr) if bbstr else str(None)) + "\n") text_data_output.write("Tesseract Data:" + "\n") for key, val in tester.text_detector.tessdata.items(): text_data_output.write("\t" + str(key) + ": " + str(val) + "\n") file_output.write("Found") except: file_output.write("Crash") crash = True file_output.write(",") if not crash and self.plot_text: plot_box( boxes=bboxes, image=image, waittime=0, save_img=self.plot_text, path=os.path.join(self.OUTPUT_IMGS_DIR, filename), quiet_output=True, ) return crash
def write(stream: IOBase, blob: bytearray) -> None: stream.write(blob)
def compileRun(input:IOBase, output:IOBase = None, logger = logging.getLogger()): source = _compile(input, logger) if output is not None: output.write(source.decode("UTF-8")) return llvm.run(source).decode("UTF-8")
def compile(input:IOBase, output:IOBase, logger = logging.getLogger()): output.write(_compile(input, logger).decode("UTF-8"))