def assert_file(our_file: io.FileIO, passed_file: io.FileIO): our_readable = our_file.readable() got_mode = passed_file.mode our_stat = os.fstat(our_file.fileno()) passed_stat = os.fstat(passed_file.fileno()) is_fifo = stat.S_ISFIFO(passed_stat.st_mode) expected_mode = "wb" if our_readable else "rb" reader = our_file if our_readable else passed_file writer = passed_file if our_readable else our_file # Verify that we have a pipe with its two ends if is_fifo and our_stat == passed_stat: pipe_size = fcntl.fcntl(writer.fileno(), F_GETPIPE_SZ) # Check for pending bytes in the pipe pending_bytes = bytearray(SIZEOF_INT) fcntl.ioctl(reader.fileno(), termios.FIONREAD, pending_bytes) pending_bytes = struct.unpack_from("=i", pending_bytes)[0] test_size = min(mmap.PAGESIZE, pipe_size - pending_bytes) expected_bytes = random.randbytes(test_size) writer.write(expected_bytes) writer.flush() got_bytes = reader.read(pipe_size) else: expected_bytes = None got_bytes = None assert (got_mode, is_fifo, passed_stat, got_bytes) == (expected_mode, True, our_stat, expected_bytes)
def build_blob(bin1: FileIO, bin2: FileIO, out: FileIO) -> None: """Combines two firmware binary blobs ``bin1`` and ``bin2`` into a single firmware blob and saves them in ``out``. """ bin1_size = os.fstat(bin1.fileno()).st_size bin2_size = os.fstat(bin2.fileno()).st_size bin2_offset = BIN2_BASE_OFFSET - BIN1_BASE_OFFSET size = bin2_offset + bin2_size max_size = FLASH_SIZE - BIN1_BASE_OFFSET if bin1_size >= bin2_offset: raise ValueError(f"{bin1.name} is too big!") if size >= max_size: raise ValueError(f"{bin2.name} is too big!") # Create a new combined firmware blob blob = memoryview(bytearray(size)) bin1.readinto(blob) bin2.readinto(blob[bin2_offset:]) # Read Reset_Handler pointers in vector tables. bin1_reset_handler = blob[4:8].tobytes() bin2_reset_handler = blob[bin2_offset + 4 : bin2_offset + 8].tobytes() # Swap Reset_Handler pointers. This will cause the second # firmware to boot first. blob[4:8] = bin2_reset_handler blob[bin2_offset + 4 : bin2_offset + 8] = bin1_reset_handler # The final checksum is for the entire new blob # This overrides the checksum of the second firmware. blob[-4:] = crc32_checksum(BytesIO(blob), max_size).to_bytes(4, "little") out.write(blob)
def write_to_file(file: io.FileIO, data: bytes, start: int = 0): length_to_write = len(data) file.seek(start) written = 0 while written < length_to_write: written += file.write(data[written:]) os.fsync(file.fileno())
def write_to_file(file_fd: io.FileIO, dir_fileno: Optional[int], data: bytes, fsync: bool=True): length_to_write = len(data) written = 0 while written < length_to_write: written = file_fd.write(data[written:]) if fsync: fsync_file_and_dir(file_fd.fileno(), dir_fileno)
def reduce_fileio(obj: FileIO): # Needed for pytest, which sets stdout and stderr to objects that contain # unbuffered temporary files. # File descriptors duplicated across processes preserve offset and other # properties so they should remain synchronized. fd = Fd(obj.fileno()) mode = obj.mode return rebuild_fileio, (fd, mode)
def redirect_stream(stream_name: str, target_stream: io.FileIO): """Redirect a system stream to the specified file. If ``target_stream`` is None - redirect to devnull. """ if target_stream is None: target_fd = os.open(os.devnull, os.O_RDWR) else: target_fd = target_stream.fileno() system_stream = getattr(sys, stream_name) os.dup2(target_fd, system_stream.fileno()) setattr(sys, '__{}__'.format(stream_name), system_stream)
def file_flush_and_sync(f: io.FileIO): """ Call system sync, ensure write the file’s overflow_data to disk, but it's a EXPENSIVE op """ f.flush() os.fsync(f.fileno())
def _parse_cm(hf, data, progress_callback): failures = [] # is it a bytes-like? try: with memoryview(data) as mview: _parse(hf, mview, progress_callback) yield hf return except (HprofError, BufferError): # _parse failed raise except Exception as e: # pylint: disable=broad-except # we failed before calling _parse failures.append(('bytes-like?', e)) # can it be mmapped? from mmap import mmap, ACCESS_READ from io import BufferedReader import os if isinstance(data, BufferedReader): fno = data.fileno() fsize = os.fstat(fno).st_size with mmap(fno, fsize, access=ACCESS_READ) as mapped: with memoryview(mapped) as mview: _parse(hf, mview, progress_callback) yield hf return # can it be read? try: from tempfile import TemporaryFile from io import FileIO underlying_file = FileIO(data.fileno(), closefd=False) insize = os.fstat(underlying_file.fileno()).st_size with TemporaryFile() as f: buf = bytearray(256 * 1024) fsize = 0 while True: if progress_callback: progress_callback('extracting', min(underlying_file.tell(), insize - 1), insize) nread = data.readinto(buf) if not nread: break fsize += nread f.write(buf[:nread]) f.flush() if progress_callback: progress_callback('extracting', insize, insize) with mmap(f.fileno(), fsize) as mapped: with memoryview(mapped) as mview: _parse(hf, mview, progress_callback) yield hf return except BufferError as e: raise except Exception as e: # pylint: disable=broad-except prev = e while prev is not None: if isinstance(prev, HprofError): raise e prev = prev.__context__ # pylint: disable=redefined-variable-type failures.append(('tmpfile?', e)) raise TypeError('cannot handle `data` arg', data, *failures)
def streamer(): # Start by loading up available options args, other_args = util.parseopts() debug('Setting up command channel') pid = os.getpid() with open(args.pidfile, 'w') as pidfile: pidfile.write('{0}'.format(pid)) cmd_fifo = '/tmp/rscad_streamer_{0}'.format(pid) if os.path.exists(cmd_fifo): # This shouldn't happen try to rm it os.unlink(cmd_fifo) os.mkfifo(cmd_fifo) cmd_chan = FileIO(cmd_fifo, 'r+') # Load up plugins and parse plugin specific command line opts debug('loading plugins') plugin_args = loadPlugins(args.path, args.plugins, other_args) # get an appropriate rscad object debug('making rscad obj') RSCAD = rscad.rscadfactory(args.rscad, args.ffile) debug('RSCAD: %s' % (type(RSCAD))) ## Need a (e)poll object - plugins implement input # If we're on linux, use epoll's level triggered event interface, # it's a fast poll()! try: poller = select.epoll() except AttributeError: # Not on linux, use poll() try: poller = select.poll() except: # Don't have poll() either? Quit using windows! print('Must be run a platform that supports poll() or epoll()') # Add the command channel to the poller poller.register(cmd_chan.fileno(), select.POLLIN) # Init plugins - set up (e)poll for cases that care [poller.register(fileno, select.POLLIN) for fileno in [ p.init(plugin_args) for p in RSCADPlugin.plugins] if fileno is not None] ## get any plugin commands debug('Registering plugin specific commands') pcommands = dict() [pcommands.update(p.register_commands()) for p in RSCADPlugin.plugins] # Need to write rscad script to RSCAD before starting the event loop ## Hook up piping RSCAD.connect() ## main loop try: debug('starting main loop') while True: debug('Looping...') ## read script file until EOF, pumping it to RSCAD rscad_file = RSCAD.makefile() while True: line = args.script.readline() if line == '': # rewind the file for the next pass try: args.script.seek(0, 0) except IOError: # probably stdin pass break rscad_file.write(line) rscad_file.flush() ## Wait for sequence point for line in RSCAD.waitforsync('seq1'): debug('loop line: %s' % (line)) [p.handle_output(line) for p in RSCADPlugin.plugins] # check for incomming data fd = poller.poll(0) debug('Got filedes {0}'.format(fd)) if cmd_chan.fileno() in [fdes[0] for fdes in fd]: handle_command(cmd_chan, pcommands) else: # loop through plugins calling handle_data # it's up to the plugin to make sure the data belongs to it [[p.handle_input(filedes[0], RSCAD) for p in RSCADPlugin.plugins] for filedes in fd] time.sleep(args.sleeptime) finally: debug('Cleaning up') cmd_chan.close() os.unlink(cmd_chan.name) [p.cleanup() for p in RSCADPlugin.plugins] util.cleanup(RSCAD, args.script) os.unlink(args.pidfile)
def streamer(): # Start by loading up available options args, other_args = util.parseopts() ## Need the main loop right away main_loop = pyev.default_loop(debug=args.debug) try: debug('Setting up command channel') pid = os.getpid() try: with open(args.pidfile, 'w') as pidfile: pidfile.write('{0}'.format(pid)) except IOError, e: if e.errno == 13: print "Permission denied openeing pidfile: {0}".format( args.pidfile) sys.exit(e.errno) cmd_fifo = '/tmp/rscad_streamer_{0}'.format(pid) if os.path.exists(cmd_fifo): # This shouldn't happen try to rm it os.unlink(cmd_fifo) os.mkfifo(cmd_fifo) cmd_chan = FileIO(cmd_fifo, 'r+') # Load up plugins and parse plugin specific command line opts debug('loading plugins') plugin_args = loadPlugins(args.path, args.plugins, other_args) # need these, even if empty hooks = { 'plugin_commands': dict(), 'cleanup' : list(), 'input' : list(), 'output' : list(), 'filters' : list(), } # Add the command channel to the poller w = pyev.Io(cmd_chan.fileno(), pyev.EV_READ, main_loop, handle_command, data=[cmd_chan, hooks['plugin_commands']]) w.start() for p in RSCADPlugin.plugins: r = p.init(plugin_args) # Extract hooks if r.has_key('input'): hooks['input'].append(r['input']) if r.has_key('output'): hooks['output'].append(r['output']) if r.has_key('commands'): hooks['plugin_commands'].update(r['commands']) if r.has_key('cleanup'): hooks['cleanup'].append(r['cleanup']) if r.has_key('filter'): hooks['filters'].append(r['filter']) ## Setup rscad obj debug('making rscad obj') RSCAD = rscad.rscadfactory(args.rscad, hooks) debug('RSCAD type: {0}'.format(type(RSCAD))) # if debug: w = pyev.Signal(signal.SIGINT, main_loop, ctlc) w.start() # else: # #daemonize # pass main_loop.data = RSCAD debug('Starting main loop') main_loop.start() debug('FOOOOO')
def file_flush_and_sync(f: io.FileIO): # If you’re starting with a buffered Python file object f, first do f.flush(), and then do os.fsync(f.fileno()), to ensure that all internal buffers associated with f are written to disk. f.flush() os.fsync(f.fileno())