def assert_file(our_file: io.FileIO, passed_file: io.FileIO):
our_readable = our_file.readable()
got_mode = passed_file.mode
our_stat = os.fstat(our_file.fileno())
passed_stat = os.fstat(passed_file.fileno())
is_fifo = stat.S_ISFIFO(passed_stat.st_mode)
expected_mode = "wb" if our_readable else "rb"
reader = our_file if our_readable else passed_file
writer = passed_file if our_readable else our_file
# Verify that we have a pipe with its two ends
if is_fifo and our_stat == passed_stat:
pipe_size = fcntl.fcntl(writer.fileno(), F_GETPIPE_SZ)
# Check for pending bytes in the pipe
pending_bytes = bytearray(SIZEOF_INT)
fcntl.ioctl(reader.fileno(), termios.FIONREAD, pending_bytes)
pending_bytes = struct.unpack_from("=i", pending_bytes)[0]
test_size = min(mmap.PAGESIZE, pipe_size - pending_bytes)
expected_bytes = random.randbytes(test_size)
writer.write(expected_bytes)
writer.flush()
got_bytes = reader.read(pipe_size)
else:
expected_bytes = None
got_bytes = None
assert (got_mode, is_fifo, passed_stat,
got_bytes) == (expected_mode, True, our_stat, expected_bytes)
def build_blob(bin1: FileIO, bin2: FileIO, out: FileIO) -> None:
"""Combines two firmware binary blobs ``bin1`` and ``bin2`` into a single
firmware blob and saves them in ``out``.
"""
bin1_size = os.fstat(bin1.fileno()).st_size
bin2_size = os.fstat(bin2.fileno()).st_size
bin2_offset = BIN2_BASE_OFFSET - BIN1_BASE_OFFSET
size = bin2_offset + bin2_size
max_size = FLASH_SIZE - BIN1_BASE_OFFSET
if bin1_size >= bin2_offset:
raise ValueError(f"{bin1.name} is too big!")
if size >= max_size:
raise ValueError(f"{bin2.name} is too big!")
# Create a new combined firmware blob
blob = memoryview(bytearray(size))
bin1.readinto(blob)
bin2.readinto(blob[bin2_offset:])
# Read Reset_Handler pointers in vector tables.
bin1_reset_handler = blob[4:8].tobytes()
bin2_reset_handler = blob[bin2_offset + 4 : bin2_offset + 8].tobytes()
# Swap Reset_Handler pointers. This will cause the second
# firmware to boot first.
blob[4:8] = bin2_reset_handler
blob[bin2_offset + 4 : bin2_offset + 8] = bin1_reset_handler
# The final checksum is for the entire new blob
# This overrides the checksum of the second firmware.
blob[-4:] = crc32_checksum(BytesIO(blob), max_size).to_bytes(4, "little")
out.write(blob)
def write_to_file(file: io.FileIO, data: bytes, start: int = 0):
length_to_write = len(data)
file.seek(start)
written = 0
while written < length_to_write:
written += file.write(data[written:])
os.fsync(file.fileno())
def write_to_file(file_fd: io.FileIO, dir_fileno: Optional[int],
data: bytes, fsync: bool=True):
length_to_write = len(data)
written = 0
while written < length_to_write:
written = file_fd.write(data[written:])
if fsync:
fsync_file_and_dir(file_fd.fileno(), dir_fileno)
def reduce_fileio(obj: FileIO):
# Needed for pytest, which sets stdout and stderr to objects that contain
# unbuffered temporary files.
# File descriptors duplicated across processes preserve offset and other
# properties so they should remain synchronized.
fd = Fd(obj.fileno())
mode = obj.mode
return rebuild_fileio, (fd, mode)
def redirect_stream(stream_name: str, target_stream: io.FileIO):
"""Redirect a system stream to the specified file.
If ``target_stream`` is None - redirect to devnull.
"""
if target_stream is None:
target_fd = os.open(os.devnull, os.O_RDWR)
else:
target_fd = target_stream.fileno()
system_stream = getattr(sys, stream_name)
os.dup2(target_fd, system_stream.fileno())
setattr(sys, '__{}__'.format(stream_name), system_stream)
def redirect_stream(stream_name: str, target_stream: io.FileIO):
"""Redirect a system stream to the specified file.
If ``target_stream`` is None - redirect to devnull.
"""
if target_stream is None:
target_fd = os.open(os.devnull, os.O_RDWR)
else:
target_fd = target_stream.fileno()
system_stream = getattr(sys, stream_name)
os.dup2(target_fd, system_stream.fileno())
setattr(sys, '__{}__'.format(stream_name), system_stream)
def file_flush_and_sync(f: io.FileIO):
"""
Call system sync, ensure write the file’s overflow_data to disk, but it's a EXPENSIVE op
"""
f.flush()
os.fsync(f.fileno())
def _parse_cm(hf, data, progress_callback):
failures = []
# is it a bytes-like?
try:
with memoryview(data) as mview:
_parse(hf, mview, progress_callback)
yield hf
return
except (HprofError, BufferError):
# _parse failed
raise
except Exception as e: # pylint: disable=broad-except
# we failed before calling _parse
failures.append(('bytes-like?', e))
# can it be mmapped?
from mmap import mmap, ACCESS_READ
from io import BufferedReader
import os
if isinstance(data, BufferedReader):
fno = data.fileno()
fsize = os.fstat(fno).st_size
with mmap(fno, fsize, access=ACCESS_READ) as mapped:
with memoryview(mapped) as mview:
_parse(hf, mview, progress_callback)
yield hf
return
# can it be read?
try:
from tempfile import TemporaryFile
from io import FileIO
underlying_file = FileIO(data.fileno(), closefd=False)
insize = os.fstat(underlying_file.fileno()).st_size
with TemporaryFile() as f:
buf = bytearray(256 * 1024)
fsize = 0
while True:
if progress_callback:
progress_callback('extracting',
min(underlying_file.tell(), insize - 1),
insize)
nread = data.readinto(buf)
if not nread:
break
fsize += nread
f.write(buf[:nread])
f.flush()
if progress_callback:
progress_callback('extracting', insize, insize)
with mmap(f.fileno(), fsize) as mapped:
with memoryview(mapped) as mview:
_parse(hf, mview, progress_callback)
yield hf
return
except BufferError as e:
raise
except Exception as e: # pylint: disable=broad-except
prev = e
while prev is not None:
if isinstance(prev, HprofError):
raise e
prev = prev.__context__ # pylint: disable=redefined-variable-type
failures.append(('tmpfile?', e))
raise TypeError('cannot handle `data` arg', data, *failures)
def streamer():
# Start by loading up available options
args, other_args = util.parseopts()
debug('Setting up command channel')
pid = os.getpid()
with open(args.pidfile, 'w') as pidfile:
pidfile.write('{0}'.format(pid))
cmd_fifo = '/tmp/rscad_streamer_{0}'.format(pid)
if os.path.exists(cmd_fifo):
# This shouldn't happen try to rm it
os.unlink(cmd_fifo)
os.mkfifo(cmd_fifo)
cmd_chan = FileIO(cmd_fifo, 'r+')
# Load up plugins and parse plugin specific command line opts
debug('loading plugins')
plugin_args = loadPlugins(args.path, args.plugins, other_args)
# get an appropriate rscad object
debug('making rscad obj')
RSCAD = rscad.rscadfactory(args.rscad, args.ffile)
debug('RSCAD: %s' % (type(RSCAD)))
## Need a (e)poll object - plugins implement input
# If we're on linux, use epoll's level triggered event interface,
# it's a fast poll()!
try:
poller = select.epoll()
except AttributeError:
# Not on linux, use poll()
try:
poller = select.poll()
except:
# Don't have poll() either? Quit using windows!
print('Must be run a platform that supports poll() or epoll()')
# Add the command channel to the poller
poller.register(cmd_chan.fileno(), select.POLLIN)
# Init plugins - set up (e)poll for cases that care
[poller.register(fileno, select.POLLIN) for fileno in [
p.init(plugin_args) for p in RSCADPlugin.plugins] if
fileno is not None]
## get any plugin commands
debug('Registering plugin specific commands')
pcommands = dict()
[pcommands.update(p.register_commands()) for p in RSCADPlugin.plugins]
# Need to write rscad script to RSCAD before starting the event loop
## Hook up piping
RSCAD.connect()
## main loop
try:
debug('starting main loop')
while True:
debug('Looping...')
## read script file until EOF, pumping it to RSCAD
rscad_file = RSCAD.makefile()
while True:
line = args.script.readline()
if line == '':
# rewind the file for the next pass
try:
args.script.seek(0, 0)
except IOError:
# probably stdin
pass
break
rscad_file.write(line)
rscad_file.flush()
## Wait for sequence point
for line in RSCAD.waitforsync('seq1'):
debug('loop line: %s' % (line))
[p.handle_output(line) for p in RSCADPlugin.plugins]
# check for incomming data
fd = poller.poll(0)
debug('Got filedes {0}'.format(fd))
if cmd_chan.fileno() in [fdes[0] for fdes in fd]:
handle_command(cmd_chan, pcommands)
else:
# loop through plugins calling handle_data
# it's up to the plugin to make sure the data belongs to it
[[p.handle_input(filedes[0], RSCAD) for p in
RSCADPlugin.plugins] for filedes in fd]
time.sleep(args.sleeptime)
finally:
debug('Cleaning up')
cmd_chan.close()
os.unlink(cmd_chan.name)
[p.cleanup() for p in RSCADPlugin.plugins]
util.cleanup(RSCAD, args.script)
os.unlink(args.pidfile)
def streamer():
# Start by loading up available options
args, other_args = util.parseopts()
## Need the main loop right away
main_loop = pyev.default_loop(debug=args.debug)
try:
debug('Setting up command channel')
pid = os.getpid()
try:
with open(args.pidfile, 'w') as pidfile:
pidfile.write('{0}'.format(pid))
except IOError, e:
if e.errno == 13:
print "Permission denied openeing pidfile: {0}".format(
args.pidfile)
sys.exit(e.errno)
cmd_fifo = '/tmp/rscad_streamer_{0}'.format(pid)
if os.path.exists(cmd_fifo):
# This shouldn't happen try to rm it
os.unlink(cmd_fifo)
os.mkfifo(cmd_fifo)
cmd_chan = FileIO(cmd_fifo, 'r+')
# Load up plugins and parse plugin specific command line opts
debug('loading plugins')
plugin_args = loadPlugins(args.path, args.plugins, other_args)
# need these, even if empty
hooks = {
'plugin_commands': dict(),
'cleanup' : list(),
'input' : list(),
'output' : list(),
'filters' : list(),
}
# Add the command channel to the poller
w = pyev.Io(cmd_chan.fileno(), pyev.EV_READ, main_loop, handle_command,
data=[cmd_chan, hooks['plugin_commands']])
w.start()
for p in RSCADPlugin.plugins:
r = p.init(plugin_args)
# Extract hooks
if r.has_key('input'):
hooks['input'].append(r['input'])
if r.has_key('output'):
hooks['output'].append(r['output'])
if r.has_key('commands'):
hooks['plugin_commands'].update(r['commands'])
if r.has_key('cleanup'):
hooks['cleanup'].append(r['cleanup'])
if r.has_key('filter'):
hooks['filters'].append(r['filter'])
## Setup rscad obj
debug('making rscad obj')
RSCAD = rscad.rscadfactory(args.rscad, hooks)
debug('RSCAD type: {0}'.format(type(RSCAD)))
# if debug:
w = pyev.Signal(signal.SIGINT, main_loop, ctlc)
w.start()
# else:
# #daemonize
# pass
main_loop.data = RSCAD
debug('Starting main loop')
main_loop.start()
debug('FOOOOO')
def file_flush_and_sync(f: io.FileIO):
# If you’re starting with a buffered Python file object f, first do f.flush(), and then do os.fsync(f.fileno()), to ensure that all internal buffers associated with f are written to disk.
f.flush()
os.fsync(f.fileno())
