def read_until(self,
min_num_bytes,
ending,
timeout=10,
data_consumer=None):
"""Read from board until 'ending'. Timeout None disables timeout."""
dprint("read_until({}, {})".format(min_num_bytes, ending))
data = self.read(min_num_bytes)
if data_consumer:
data_consumer(data)
timeout_count = 0
while True:
if data.endswith(ending):
dprint(" data = '{}'".format(data))
break
elif self.in_waiting > 0:
new_data = self.read(1)
data = data + new_data
if data_consumer:
data_consumer(new_data)
timeout_count = 0
else:
timeout_count += 1
if timeout and timeout_count >= 100 * timeout:
raise ConnectionError(
'timeout in read_until "{}"'.format(ending))
time.sleep(0.01)
return data
def remote(self, func, *args, xfer_func=None, **kwargs):
"""Call func with args on the micropython board."""
has_buffer = self._has_buffer
buffer_size = self.get_config('buffer_size', default=128)
time_offset = self.get_config('time_offset', default=946684800)
set_fileops_params(has_buffer, buffer_size, time_offset)
args_arr = [self._remote_repr(i) for i in args]
kwargs_arr = [
"{}={}".format(k, self._remote_repr(v)) for k, v in kwargs.items()
]
func_str = inspect.getsource(func)
func_str += 'output = ' + func.__name__ + '('
func_str += ', '.join(args_arr + kwargs_arr)
func_str += ')\n'
func_str += 'if output is None:\n'
func_str += ' print("None")\n'
func_str += 'else:\n'
func_str += ' print(output)\n'
func_str = func_str.replace('TIME_OFFSET', '{}'.format(time_offset))
func_str = func_str.replace('HAS_BUFFER', '{}'.format(has_buffer))
func_str = func_str.replace('BUFFER_SIZE', '{}'.format(buffer_size))
func_str = func_str.replace('IS_UPY', 'True')
start_time = time.time()
output = self._exec_no_output(func_str)
if xfer_func:
xfer_func(self, *args, **kwargs)
output = self._exec_output()
dprint("remote: {}({}) --> {}, in {:.3} s)".format(
func.__name__,
repr(args)[1:-1], output,
time.time() - start_time))
return output
def disconnect(self):
"""Disconnect and release port / ip"""
dprint("Disconnecting board", self._id)
self._id = None
self._root_dirs = []
if self._serial:
self._serial.close()
self._serial = None
def remove(self, board_id, option):
"""Remove board option or entire record if option=None."""
if board_id == 0:
board_id = 'default'
dprint("config.remove id={} option={}".format(board_id, option))
try:
self._modified = True
del self._boards()[board_id][option]
except KeyError:
pass
def filename_complete(self, text, line, begidx, endidx):
"""Wrapper for catching exceptions since cmd seems to silently
absorb them.
"""
try:
completion = self.real_filename_complete(text, line, begidx,
endidx)
dprint(
f"filename completion, text={text}, line={line}, begidx={begidx}, endidx={endidx}"
)
return completion
except:
traceback.print_exc()
def _exec_no_output(self, cmd, data_consumer=None, timeout=10):
"""Send command (string or bytes) to board for execution.
Pass board output to data_consumer (e.g. print).
no_output ... won't get execution output."""
if isinstance(cmd, str):
cmd = bytes(cmd, encoding='utf-8')
dprint()
dprint("_exec_no_output:", cmd.decode('utf-8')[:20], "...")
# enter raw repl (if needed) and check if we have a prompt
self.enter_raw_repl()
dprint("wait for >")
data = self._serial.read_until(1, b'>', timeout=1)
if not data.endswith(b'>'):
raise BoardError("Cannot get response from board")
# send command to board
for i in range(0, len(cmd), 256):
self._serial.write(cmd[i:min(i + 256, len(cmd))])
time.sleep(0.01)
# execute command
self._serial.write(b'\x04')
# check if successful
if self._serial.read(2) != b'OK':
self._status = self.STATUS_UNKNOWN
raise BoardError("Could not exec '{} ...'".format(
cmd.decode('utf-8').partition('\n')[0]))
dprint("_exec_no_output done")
def __init__(self, ip, user, password, read_timeout=5):
dprint("TelnetConnection({}, user={}, password={})".format(
ip, user, password))
import telnetlib
try:
self._telnet = telnetlib.Telnet(ip, timeout=15)
except ConnectionRefusedError:
raise ConnectionError("Board refused telnet connection")
self._ip = ip
self._read_timeout = read_timeout
if b'Login as:' in self._telnet.read_until(b'Login as:',
timeout=read_timeout):
self._telnet.write(bytes(user, 'ascii') + b"\r\n")
dprint("sent user", user)
if b'Password:'******'Password:'******'ascii') + b"\r\n")
dprint("sent password", password)
if b'for more information.' in self._telnet.read_until(
b'Type "help()" for more information.',
timeout=read_timeout):
dprint("got greeting")
# login succesful
from collections import deque
self._fifo = deque()
return
raise ConnectionError(
'Failed to establish a telnet connection with the board')
def attach_commands():
"""Import commands defined in folder do/"""
dir = os.path.dirname(inspect.getfile(inspect.currentframe()))
dir = os.path.join(dir, 'do')
for filename in os.listdir(dir):
if not filename.startswith('do_') or not filename.endswith('.py'):
continue
module_name = os.path.splitext(filename)[0]
cmd_name = module_name[3:]
module = importlib.import_module(module_name)
# attach to the shell
dprint("attaching command", module_name, end='')
setattr(Shell, module_name, getattr(module, module_name))
try:
argparse = "argparse_" + cmd_name
setattr(Shell, argparse, getattr(module, argparse))
dprint(" +", argparse, end='')
except AttributeError:
pass
try:
complete = "complete_" + cmd_name
setattr(Shell, complete, getattr(module, complete))
dprint(" +", complete, end='')
except AttributeError:
pass
dprint()
def do_flash(self, line):
"""flash [-l|--list] [-e|--erase] [-v|--version VERSION] [-b|--board BOARD]
Flash firmware to microcontroller.
"""
args = self.line_to_args(line)
firmware_url = "https://people.eecs.berkeley.edu/~boser/iot49/firmware"
flash_options = "--chip esp32 " \
"--before default_reset --after hard_reset " \
"write_flash -z --flash_mode dio --flash_freq 40m --flash_size detect"
id = 0
try:
id = self.boards.default.id
except BoardError:
pass
firmware_url = self.config.get(id, "firmware_url", firmware_url)
flash_options = self.config.get(id, "flash_options", flash_options)
port = self.config.get(id, "port", "/dev/cu.SLAB_USBtoUART")
baudrate = self.config.get(id, "flash_baudrate", 921600)
board = self.config.get(id, "board", "HUZZAH32")
if args.board:
board = args.board
dprint("firmware url: ", firmware_url)
dprint("flash options:", flash_options)
dprint("port: ", port)
dprint("baudrate: ", baudrate)
dprint("board: ", board)
try:
f = Flasher(board=board, url=firmware_url)
if args.list:
print("available firmware versions:")
print('\n'.join(
[" {:8s} {}".format(v, d) for v, d in f.versions()]))
return
dev = self.boards.find_board(port)
if dev: dev.disconnect()
if args.erase:
f.erase_flash(port)
f.flash(args.version,
flash_options=flash_options,
port=port,
baudrate=baudrate)
except FlasherError as e:
eprint(e)
def enter_raw_repl_cp(self):
"""Enter raw repl if not already in this mode for CIRCUITPYTHON."""
# Ctrl-C twice: interrupt any running program
dprint("^C, abort running program")
self._serial.write(b'\r\x03\x03')
# Ctrl-A: enter raw REPL
dprint("^A, raw repl")
self._serial.write(b'\r\x01')
expect = b"raw REPL; CTRL-B to exit"
data = self._serial.read_until(1, expect)
if not data.endswith(expect):
raise BoardError('Cannot enter raw repl: expected {}, got {}'.format(expect, data))
expect = b"\r\n"
data = self._serial.read_until(1, expect)
if not data.endswith(expect):
raise BoardError('Cannot enter raw repl: expected {}, got {}'.format(expect, data))
def file_dir(devs, directory):
"""Dict name->stat of files in directory,
filted by rsync_includes, rsync_excludes
"""
dev, filename = devs.get_dev_and_path(directory)
inc = devs.config.get(0, 'rsync_includes',
default='*.py,*.json,*.txt,*.html').split(',')
exc = devs.config.get(0, 'rsync_excludes', default='.DS_store,__*__').split(',')
files = auto(devs, listdir_stat, directory)
if not files:
files = []
d = {}
for name, stat in files:
y = any(map((lambda x: fnmatch.fnmatch(name, x)), inc)) or is_dir(stat)
n = any(map((lambda x: fnmatch.fnmatch(name, x)), exc))
if y and not n:
d[name] = stat
else:
dprint("squashing {} y={} n={}".format(name, y, n))
return d
def set(self, board_id, option, value):
"""Set board option parameter value. board_id = 0 is default entries."""
dprint("config.set id={} {}={}".format(board_id, option, value))
if board_id == 0:
board_id = 'default'
if not option:
return
if not isinstance(option, str):
raise ConfigError(
"{}: expected str, got {!r}".format(option, type(option)))
if not option.isidentifier():
raise ConfigError(
"{} is not a valid Python identifier".format(option))
if keyword.iskeyword(option):
raise ConfigError(
"{}: keywords are not permitted as option names".format(option))
self._modified = True
boards = self._boards()
if not board_id in boards:
boards[board_id] = {}
boards[board_id][option] = value
def enter_raw_repl_mp(self):
"""Enter raw repl if not already in this mode."""
# Ctrl-C twice: interrupt any running program
dprint("^C, abort running program")
self._serial.write(b'\r\x03\x03')
time.sleep(0.1)
# discard any waiting input
dprint("purge in_waiting")
while self._serial.in_waiting:
self._serial.read(self._serial.in_waiting)
time.sleep(0.1)
# Ctrl-A: enter raw REPL
dprint("^A, raw repl")
self._serial.write(b'\r\x01')
expect = b'raw REPL; CTRL-B to exit\r\n'
data = self._serial.read_until(1, expect)
if not data.endswith(expect):
raise BoardError(
'Cannot enter raw repl: expected {}, got {}'.format(
expect, data))
time.sleep(0.1)
# determine required steps
# Note 1: no soft reset breaks telnet connection
# Note 2: if user pressed reset button, mode status is STATUS_NORMAL_REPL
# but shell49 won't know it. Hence we cannot assume RAW_REPL.
# BUT soft reset is not required.
if self.is_telnet or self._status == self.STATUS_RAW_REPL:
dprint("enter_raw_repl: already in RAW REPL state, no action")
return
# Ctrl-D: soft reset
dprint("^D, soft reset")
self._serial.write(b'\x04')
expect = b'soft reboot\r\n'
data = self._serial.read_until(1, expect)
if not data.endswith(expect):
raise BoardError(
'Could not do soft reset: expected {}, got {}'.format(
expect, data))
# By splitting this into 2 reads, it allows boot.py to print stuff,
# which will show up after the soft reboot and before the raw REPL.
# The next read_until takes ~0.8 seconds (on ESP32)
expect = b'raw REPL; CTRL-B to exit\r\n'
data = self._serial.read_until(1, expect)
if not data.endswith(expect):
raise BoardError('Soft reset failed: expected {}, got {}'.format(
expect, data))
# update board status
self._status = self.STATUS_RAW_REPL
dprint("in raw repl")
def rsync(devs, src_dir, dst_dir, mirror, dry_run, recursed):
"""Synchronizes 2 directory trees."""
# This test is a hack to avoid errors when accessing /flash. When the
# cache synchronisation issue is solved it should be removed
if not isinstance(src_dir, str) or not len(src_dir):
return
# check that source is a directory
sstat = auto(devs, get_stat, src_dir)
if not is_dir(sstat):
eprint("*** Source {} is not a directory".format(src_dir))
return
# create destination directory if it does not exist
sstat = auto(devs, get_stat, dst_dir)
if not file_exists(sstat):
qprint("Create {} on remote".format(dst_dir))
if not dry_run:
if recursed and not make_dir(devs, dst_dir, dry_run, recursed):
eprint("*** Unable to create directory", dst_dir)
elif not is_dir(sstat):
eprint("*** Destination {} is not a directory".format(src_dir))
return
# get list of src & dst files and stats
qprint(" checking {}".format(dst_dir))
d_src = file_dir(devs, src_dir)
d_dst = file_dir(devs, dst_dir)
# determine what needs to be copied or deleted
set_dst = set(d_dst.keys())
set_src = set(d_src.keys())
to_add = set_src - set_dst # Files to copy to dest
to_del = set_dst - set_src # To delete from dest
to_upd = set_dst.intersection(set_src) # In both: may need updating
if False:
eprint("rsync {} -> {}".format(src_dir, dst_dir))
eprint(" sources", set_src)
eprint(" dest ", set_dst)
eprint(" add ", to_add)
eprint(" delete ", to_del)
eprint(" update ", to_upd)
# add ...
for f in to_add:
src = os.path.join(src_dir, f)
dst = os.path.join(dst_dir, f)
qprint("Adding {}".format(dst))
if is_dir(d_src[f]):
if recursed:
rsync(devs, src, dst, mirror, dry_run, recursed)
else:
if not dry_run:
if not cp(devs, src, dst):
eprint("*** Unable to add {} --> {}".format(src, dst))
# delete ...
for f in to_del:
if not mirror:
break
dst = os.path.join(dst_dir, f)
qprint("Removing {}".format(dst))
if not dry_run:
res = rm(devs, dst, recursive=True, force=True)
if not res:
eprint("Cannot remove {}", dst)
# update ...
for f in to_upd:
src = os.path.join(src_dir, f)
dst = os.path.join(dst_dir, f)
if is_dir(d_src[f]):
if is_dir(d_dst[f]):
# src and dst are directories
if recursed:
rsync(devs, src, dst, mirror, dry_run, recursed)
else:
msg = "Source '{}' is a directory and destination " \
"'{}' is a file. Ignoring"
eprint(msg.format(src, dst))
else:
if is_dir(d_dst[f]):
msg = "Source '{}' is a file and destination " \
"'{}' is a directory. Ignoring"
eprint(msg.format(src, dst))
else:
if False:
eprint("BEB src {} > dst {} delta={}".format(
stat_mtime(d_src[f]), stat_mtime(d_dst[f]),
stat_mtime(d_src[f]) - stat_mtime(d_dst[f])))
if stat_size(d_src[f]) != stat_size(d_dst[f]) or \
stat_mtime(d_src[f]) > stat_mtime(d_dst[f]):
msg = "Copying {} (newer than {})"
qprint(msg.format(src, dst))
if not dry_run:
if not cp(devs, src, dst):
eprint(
"*** Unable to update {} --> {}".format(src, dst))
else:
dprint(f, "NO update src time:", stat_mtime(d_src[f]), "dst time", stat_mtime(
d_dst[f]), "delta", stat_mtime(d_src[f]) - stat_mtime(d_dst[f]))
def enter_raw_repl_mp(self):
"""Enter raw repl if not already in this mode for MICROPYTHON."""
dprint("^B^C, abort running program")
self._serial.write(b'\r\x02\x03')
time.sleep(.1)
# Attempt to get to REPL prompt, send Ctrl-C on failure.
expect = b'> '
abort = True
for attempt in range(3):
try:
self._serial.read_until(1, expect)
abort = False
break
except ConnectionError as err:
dprint('ConnectionError: {0}'.format(err))
self._serial.write(b'\x03')
time.sleep(1)
# Kickout if 3rd attempt fails
if abort:
raise ConnectionError('Failed to enter raw REPL')
time.sleep(.1)
# Ctrl-A: enter raw REPL
dprint("^A, raw repl")
self._serial.write(b'\r\x01')
expect = b'raw REPL; CTRL-B to exit\r\n'
data = self._serial.read_until(1, expect)
if not data.endswith(expect):
raise BoardError('Cannot enter raw repl: expected {}, got {}'.format(expect, data))
# determine required steps
# Note 1: no soft reset breaks telnet connection
# Note 2: if user pressed reset button, mode status is STATUS_NORMAL_REPL
# but shell49 won't know it. Hence we cannot assume RAW_REPL.
# BUT soft reset is not required.
if self.is_telnet or self._status == self.STATUS_RAW_REPL:
dprint("enter_raw_repl: already in RAW REPL state, no action")
return
# Ctrl-D: soft reset
dprint("^D, soft reset")
self._serial.write(b'\x04')
expect = b'soft reboot\r\n'
data = self._serial.read_until(1, expect)
if not data.endswith(expect):
raise BoardError('Could not do soft reset: expected {}, got {}'.format(expect, data))
# By splitting this into 2 reads, it allows boot.py to print stuff,
# which will show up after the soft reboot and before the raw REPL.
# The next read_until takes ~0.8 seconds (on ESP32)
expect = b'raw REPL; CTRL-B to exit\r\n'
data = self._serial.read_until(1, expect)
if not data.endswith(expect):
raise BoardError('Soft reset failed: expected {}, got {}'.format(expect, data))
# update board status
self._status = self.STATUS_RAW_REPL
dprint("in raw repl")
def main():
"""The main program."""
if sys.version_info.major < 3:
v = sys.version_info
eprint("Shell49 requires Python 3.6 (not {}.{}.{})".format(
v.major, v.minor, v.micro))
return
default_config = os.getenv('SHELL49_CONFIG_FILE') or '~/.shell49_rc.py'
default_editor = os.getenv('SHELL49_EDITOR') or os.getenv(
'VISUAL') or os.getenv('EDITOR') or 'vi'
default_nocolor = 'win32' in sys.platform
default_debug = False
default_quiet = False
parser = argparse.ArgumentParser(
prog="shell49",
usage="%(prog)s [options] [cmd]",
description="Remote Shell for MicroPython boards.",
epilog=("""
Environment variables:
SHELL49_CONFIG_FILE configuration file (Default: '{}')
SHELL49_EDITOR editor (Default: {})
""".format(default_config, default_editor)),
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument(
"-c",
"--config",
dest="config",
help="Set path of the configuration file (default: '%s')" %
default_config,
default=default_config)
parser.add_argument("-e",
"--editor",
dest="editor",
help="Set the editor to use (default: '%s')" %
default_editor,
default=default_editor)
parser.add_argument("-d",
"--debug",
dest="debug",
action="store_true",
help="Enable debug features (default %s)" %
default_debug,
default=default_debug)
parser.add_argument("-n",
"--nocolor",
dest="nocolor",
action="store_true",
help="Turn off colorized output (default: %s)" %
default_nocolor,
default=default_nocolor)
parser.add_argument("--quiet",
dest="quiet",
action="store_true",
help="Turn off some output (default: %s)" %
default_quiet,
default=False)
parser.add_argument(
"-a",
"--no_auto_connect",
dest="auto_connect",
action="store_false",
help="Do not automatically connect to board connected to serial port",
default=True)
parser.add_argument('-V',
'--version',
dest='version',
action='store_true',
help='Report the version and exit.',
default=False)
parser.add_argument("-f",
"--file",
dest="filename",
help="File of commands to process (non-interactive).")
parser.add_argument("cmd",
nargs=argparse.REMAINDER,
help="Optional command to execute and quit.")
args = parser.parse_args(sys.argv[1:])
debug(args.debug)
quiet(args.quiet or args.cmd or args.filename)
if args.nocolor: nocolor()
dprint("config = %s" % args.config)
dprint("editor = %s" % args.editor)
dprint("debug = %s" % args.debug)
dprint("quiet = %s" % args.quiet)
dprint("nocolor = %s" % args.nocolor)
dprint("auto_connect = %s" % args.auto_connect)
dprint("version = %s" % __version__)
dprint("cmd = [%s]" % ', '.join(args.cmd))
if args.version:
print(__version__)
return
cmd_line = ' '.join(args.cmd)
if not args.filename and cmd_line == '':
oprint(
"Welcome to shell49 version {}. Type 'help' for information; Control-D to exit."
.format(__version__))
args.config = os.path.expanduser(args.config)
args.config = os.path.normpath(args.config)
with Config(args.config) as config:
boards = ActiveBoards(config)
# connect to board ...
try:
if args.auto_connect:
boards.connect_serial(config.get('default', 'port'))
except (ConnectionError, BoardError) as err:
eprint(err)
except KeyboardInterrupt:
pass
# start command shell
attach_commands()
if args.filename:
with open(args.filename) as cmd_file:
shell = Shell(boards, args.editor, stdin=cmd_file)
shell.cmdloop('')
else:
if boards.num_boards() == 0:
eprint(
"No MicroPython boards connected - use the connect command to add one."
)
shell = Shell(boards, args.editor)
try:
shell.cmdloop(cmd_line)
except KeyboardInterrupt:
qprint("Bye")
print(printing.NO_COLOR)
