def print_version_info():
# The bootstrapper will print its own version, as well as that of
# the west repository itself, then exit. So if this file is being
# asked to print the version, it's because it's being run
# directly, and not via the bootstrapper.
#
# Rather than play tricks like invoking "pip show west" (which
# assumes the bootstrapper was installed via pip, the common but
# not universal case), refuse the temptation to make guesses and
# print an honest answer.
log.inf('West bootstrapper version: N/A, not run via bootstrapper')
# The running west installation.
if IN_MULTIREPO_INSTALL:
try:
desc = check_output(['git', 'describe', '--tags'],
stderr=DEVNULL,
cwd=os.path.dirname(__file__))
west_version = desc.decode(sys.getdefaultencoding()).strip()
except CalledProcessError as e:
west_version = 'unknown'
else:
west_version = 'N/A, monorepo installation'
west_src_west = os.path.dirname(__file__)
print('West repository version: {} ({})'.format(
west_version, os.path.dirname(os.path.dirname(west_src_west))))
def run(self):
while True:
time.sleep(1)
if self.timeout:
self.timeout -= 1
if not self.timeout:
inf('idle')
def delete_wan_file(filepath):
"""
Delete file on WAN if WAN is configured. This will be triggered when a file is
deleted from LAN filestorage.
"""
if not FILUXE_WAN:
deb(f'no wan configured, not deleting {filepath} on wan')
return
filestorage_path = os.path.relpath(filepath, FILUXE_LAN.root())
path = os.path.dirname(filestorage_path)
try:
if not ACTIVE_RULES['dirs'][path]['delete']:
deb(f'not deleting on wan since delete=false for {filepath}')
return
except:
pass
try:
file = os.path.basename(filepath)
rule_path = os.path.normpath(path)
dir_rules = ACTIVE_RULES['dirs'][rule_path]
if not fwd_util.filename_is_included(file, dir_rules):
inf(f'filename {file} is not in scope and will not be exported')
return
except:
deb(f'from "{FILUXE_WAN.log_path(filepath)}" deleting file {file} (no rules)'
)
fwd_util.delete_http_file(FILUXE_WAN, filestorage_path)
def export_file(filepath):
""" Use filuxe to upload the file if it first matches the include regex and
second doesn't match the exclude regex.
If the include regex and the exclude regex are both empty strings then
the file is exported.
"""
if not FILUXE_WAN:
return
path = os.path.dirname(filepath)
relpath = os.path.relpath(path, FILE_ROOT)
file = os.path.basename(filepath)
try:
dir_rules = ACTIVE_RULES['dirs'][relpath]
if not fwd_util.filename_is_included(file, dir_rules):
inf(f'filename {file} is not in scope and will not be exported')
return
deb(f'forwarding {file}')
except:
inf(f'from {relpath} uploading file {file} (no rules)')
try:
deb(f'uploading {FILUXE_LAN.log_path(filepath)}')
FILUXE_WAN.upload(filepath, os.path.join(relpath, file))
except requests.ConnectionError:
war('upload failed, WAN server is not reachable.')
except FileNotFoundError:
war(f'exception file not found, {os.path.join(relpath, file)} (internal race)'
)
def flash(self, **kwargs):
if self.bin_name is None:
raise ValueError('Cannot flash; bin_name is missing')
lines = ['r'] # Reset and halt the target
if self.erase:
lines.append('erase') # Erase all flash sectors
lines.append('loadfile {} 0x{:x}'.format(self.bin_name,
self.flash_addr))
lines.append('g') # Start the CPU
lines.append('q') # Close the connection and quit
# Don't use NamedTemporaryFile: the resulting file can't be
# opened again on Windows.
with tempfile.TemporaryDirectory(suffix='jlink') as d:
fname = os.path.join(d, 'runner.jlink')
with open(fname, 'wb') as f:
f.writelines(bytes(line + '\n', 'utf-8') for line in lines)
cmd = ([self.commander] +
['-if', self.iface,
'-speed', self.speed,
'-device', self.device,
'-CommanderScript', fname])
log.inf('Flashing Target Device')
self.check_call(cmd)
def main(argv=None):
# Makes ANSI color escapes work on Windows, and strips them when
# stdout/stderr isn't a terminal
colorama.init()
if argv is None:
argv = sys.argv[1:]
args, unknown = parse_args(argv)
for_stack_trace = 'run as "west -v ... {} ..." for a stack trace'.format(
args.command)
try:
args.handler(args, unknown)
except WestUpdated:
# West has been automatically updated. Restart ourselves to run the
# latest version, with the same arguments that we were given.
os.execv(sys.executable, [sys.executable] + sys.argv)
except KeyboardInterrupt:
sys.exit(0)
except CalledProcessError as cpe:
log.err('command exited with status {}: {}'.format(
cpe.args[0], quote_sh_list(cpe.args[1])))
if args.verbose:
raise
else:
log.inf(for_stack_trace)
except CommandContextError as cce:
log.die('command', args.command, 'cannot be run in this context:',
*cce.args)
except Exception as exc:
log.err(*exc.args, fatal=True)
if args.verbose:
raise
else:
log.inf(for_stack_trace)
def _inf(project, msg):
# Print '=== msg' (to clearly separate it from Git output). Supports the
# same (foo) shorthands as the git commands.
#
# Prints the message in green if stdout is a terminal, to clearly separate
# it from command (usually Git) output.
log.inf('=== ' + _expand_shorthands(project, msg), colorize=True)
def process_IN_CREATE(self, event):
if event.dir:
inf(f'new directory "{FILUXE_LAN.log_path(event.pathname)}"')
path = os.path.relpath(event.pathname, FILUXE_LAN.root())
calculate_rules(path)
else:
inf(f'new file "{FILUXE_LAN.log_path(event.pathname)}"')
new_file(event.pathname)
def do_run(self, args, user_args):
log.inf("Manifest path: {}\n".format(_manifest_path(args)))
for project in _all_projects(args):
log.inf('{:15} {:30} {:15} {}'.format(
project.name,
os.path.join(project.path, ''), # Add final '/' if missing
project.revision,
"(cloned)" if _cloned(project) else "(not cloned)"))
def route_delete(path):
path = safe_join(os.path.join(app.config['fileroot'], path))
try:
os.remove(path)
inf(f'deleting file "{path}"')
except OSError:
inf(f'file not found deleting "{path}"')
return '', 404
return '', 200
def _dump_runner_cached_opts(cache, runner, initial_indent, subsequent_indent):
runner_args = _get_runner_args(cache, runner)
if not runner_args:
return
log.inf('{}Cached runner-specific options:'.format(initial_indent),
colorize=True)
for arg in runner_args:
log.inf('{}{}'.format(subsequent_indent, arg))
def flash(self, **kwargs):
if self.bin_name is None:
raise ValueError('Cannot flash; bin_name is missing')
cmd = ([self.flashtool] + self.flash_addr_args + self.daparg_args +
self.target_args + self.board_args + self.flashtool_extra +
[self.bin_name])
log.inf('Flashing Target Device')
self.check_call(cmd)
def run_filesystem_observer(root):
global LOOP, WATCH_MANAGER
inf(f'starting file observer in {root}')
LOOP = asyncio.get_event_loop()
WATCH_MANAGER = pyinotify.WatchManager()
pyinotify.AsyncioNotifier(WATCH_MANAGER,
LOOP,
default_proc_fun=EventHandler())
mask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_DELETE | pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
WATCH_MANAGER.add_watch(root, mask, rec=True, auto_add=True)
signal.signal(signal.SIGINT, terminate)
signal.signal(signal.SIGTERM, terminate)
def route_upload(path):
global chunked_file_handle
time = request.args.get('time', type=float, default=0.0)
force = request.args.get('force', type=inputs.boolean, default=False)
path = safe_join(os.path.join(app.config['fileroot'], path))
if path is None:
abort(404)
try:
content_range = request.environ['HTTP_CONTENT_RANGE']
parsed_ranges = re.search(r'bytes (\d*)-(\d*)\/(\d*)', content_range)
_from, _to, _size = [int(x) for x in parsed_ranges.groups()]
deb(f'chunked upload, {_from} to {_to} ({_size}), {_to - _from + 1} bytes')
except:
content_range = None
if not content_range or _from == 0:
if os.path.exists(path):
if not force:
# if force was not given then the default is that the server refuses to rewrite an existing file
err(f'file {path} already exist, returning 403 (see --force)')
return '', 403
else:
directory = os.path.dirname(path)
if not os.path.exists(directory):
inf(f'constructing new path {directory}')
Path(directory).mkdir(parents=True, exist_ok=True)
if content_range:
if _from == 0:
try:
if chunked_file_handle.get(path):
err('internal error in upload, non closed filehandle')
chunked_file_handle[path].close()
open(path, 'w').close()
chunked_file_handle[path] = open(path, "ab")
except:
pass
inf(f'writing file "{path}" ({human_file_size(_size)})')
chunked_file_handle[path].write(request.data)
if _to == _size - 1:
inf(f'{path} transfer complete')
chunked_file_handle[path].close()
del chunked_file_handle[path]
else:
# ordinary non-chunked upload, single write
inf(f'writing file "{path}"')
with open(path, "wb") as fp:
fp.write(request.data)
if time > 0.0:
deb(f'setting {path} time to {time}')
os.utime(path, (time, time))
# 201: Created
return '', 201
def call(self, cmd):
'''Subclass subprocess.call() wrapper.
Subclasses should use this method to run command in a
subprocess and get its return code, rather than
using subprocess directly, to keep accurate debug logs.
'''
quoted = quote_sh_list(cmd)
if JUST_PRINT:
log.inf(quoted)
return 0
log.dbg(quoted)
return subprocess.call(cmd)
def do_run(self, args, user_args):
if args.branch:
# Create a branch in the specified projects
for project in _cloned_projects(args):
_create_branch(project, args.branch)
else:
# No arguments. List local branches from all cloned projects along
# with the projects they appear in.
branch2projs = collections.defaultdict(list)
for project in _cloned_projects(args):
for branch in _branches(project):
branch2projs[branch].append(project.name)
for branch, projs in sorted(branch2projs.items()):
log.inf('{:18} {}'.format(branch, ", ".join(projs)))
def download(self, filename, path, force=False):
url = f'{self.server}/download/{path}'
response = requests.get(url, verify=self.certificate)
if response.status_code != 200:
err(f'server returned error {response.status_code} for downloading "{path}"'
)
return ErrorCode.FILE_NOT_FOUND
if force or not os.path.exists(filename):
with open(filename, 'wb') as f:
f.write(response.content)
inf(f'downloaded {url} ({human_file_size(os.path.getsize(filename))}) as "{filename}"'
)
else:
die(f'local file "{filename}" already exists, bailing out (see --force)',
error_code=ErrorCode.FILE_ALREADY_EXIST)
return ErrorCode.OK
def synchonize(lan_files):
inf('synchonizing with WAN server, please wait')
with Indent() as _:
# If the WAN server is missing then the forwarder will not be able to do its job before
# the WAN server can be reached.
wan_files = fwd_util.get_http_filelist(FILUXE_WAN, rules=ACTIVE_RULES)
if lan_files is None or wan_files is None:
war('retrieving filelists failed, synchonization aborted')
return
inf(f'found {lan_files["info"]["files"]} files on LAN server and {wan_files["info"]["files"]} on WAN server'
)
new_files = []
modified_files = []
copy_bytes = 0
for directory, filelist in lan_files['filelist'].items():
if directory not in wan_files['filelist']:
for filename, metrics in lan_files['filelist'][
directory].items():
pathname = os.path.join(directory, filename)
new_files.append(pathname)
copy_bytes += metrics['size']
continue
for filename, metrics in filelist.items():
pathname = os.path.join(directory, filename)
if filename not in wan_files['filelist'][directory]:
new_files.append(pathname)
copy_bytes += metrics['size']
elif metrics['time'] != wan_files['filelist'][directory][
filename]['time']:
modified_files.append(pathname)
copy_bytes += metrics['size']
if not len(new_files) + len(modified_files):
inf('WAN server is up-to-date')
else:
inf(f'synchonizing: uploading {human_file_size(copy_bytes)} in {len(new_files)} new files '
f'and {len(modified_files)} modified files')
for file in new_files + modified_files:
export_file(os.path.join(FILE_ROOT, file))
inf('synchonizing: complete')
def check_output(self, cmd):
'''Subclass subprocess.check_output() wrapper.
Subclasses should use this method to run command in a
subprocess and check that it executed correctly, rather than
using subprocess directly, to keep accurate debug logs.
'''
quoted = quote_sh_list(cmd)
if JUST_PRINT:
log.inf(quoted)
return b''
log.dbg(quoted)
try:
return subprocess.check_output(cmd)
except subprocess.CalledProcessError:
raise
def parse_into_file_groups(self, directory, filelist, directory_settings):
file_groups = {}
max_files, delete_by, file_group_rules = directory_settings
deb(f'find groups in {self.domain.domain} "{directory}" ({len(filelist["filelist"][directory])} files)')
for filename, fileinfo in filelist['filelist'][directory].items():
group_key = 'ungrouped'
for group in file_group_rules:
try:
match = re.match(fr'{group}', filename)
except Exception as e:
err(f'regex gave exception {e.__repr__()} with regex "{group}"')
exit(1)
if match:
nof_groups = len(match.groups())
nof_group_regex_groups = re.compile(group).groups
if nof_groups != nof_group_regex_groups:
deb(f'parsing {filename} failed, found {nof_groups} groups, not {nof_group_regex_groups}')
else:
group_key = ':'.join(match.groups())
break
if file_group_rules and group_key == 'ungrouped':
inf(f'no group match for {filename}, adding to "ungrouped"')
group_key = os.path.join(directory, group_key)
try:
file_groups[group_key]
except:
file_groups[group_key] = {}
file_groups[group_key]['files'] = {}
file_groups[group_key]['maxfiles'] = max_files
file_groups[group_key]['deleteby'] = delete_by
file_groups[group_key]['directory'] = directory
file_groups[group_key]['files'][filename] = fileinfo
for group in file_groups.keys():
deb(f'group {group} with {len(file_groups[group]["files"])} files')
return file_groups
def do_run(self, command, **kwargs):
commands = []
if (self.snr is None):
board_snr = self.get_board_snr_from_user()
else:
board_snr = self.snr.lstrip("0")
program_cmd = [
'nrfjprog', '--program', self.hex_, '-f', self.family, '--snr',
board_snr
]
print('Flashing file: {}'.format(self.hex_))
if self.erase:
commands.extend([[
'nrfjprog', '--eraseall', '-f', self.family, '--snr', board_snr
], program_cmd])
else:
if self.family == 'NRF51':
commands.append(program_cmd + ['--sectorerase'])
else:
commands.append(program_cmd + ['--sectoranduicrerase'])
if self.family == 'NRF52' and self.softreset == False:
commands.extend([
# Enable pin reset
[
'nrfjprog', '--pinresetenable', '-f', self.family, '--snr',
board_snr
],
])
if self.softreset:
commands.append(
['nrfjprog', '--reset', '-f', self.family, '--snr', board_snr])
else:
commands.append([
'nrfjprog', '--pinreset', '-f', self.family, '--snr', board_snr
])
for cmd in commands:
self.check_call(cmd)
log.inf('Board with serial number {} flashed successfully.'.format(
board_snr))
def list_files(path):
recursive = request.args.get('recursive', type=inputs.boolean, default=False)
path = safe_join(os.path.join(app.config['fileroot'], path))
if not os.path.exists(path):
err(f'filelist failed, path not found "{path}"')
return 'path not found', 404
fileroot = os.path.join(app.config['fileroot'], '')
file_result = {}
dir_result = []
nof_files = 0
for _root, dirs, _files in os.walk(path):
for file in _files:
p = os.path.join(_root, file)
if not file_is_closed(p):
war(f'skipping {p} since it is busy')
continue
relative = os.path.relpath(_root, fileroot)
if not file_result.get(relative):
file_result[relative] = {}
file_result[relative][file] = {'size': os.path.getsize(p), 'time': get_file_time(p)}
for directory in dirs:
rel_path = os.path.join(os.path.relpath(_root, fileroot), directory)
dir_result.append(os.path.normpath(rel_path))
nof_files += len(_files)
if not recursive:
break
extra = "(recursive)" if recursive else ""
inf(f'returning filelist at "{path}", {nof_files} files and {len(dir_result)} directories. {extra}')
ret = {'info':
{
'fileroot': fileroot, 'files': nof_files, 'dirs': len(dir_result)
},
'filelist': file_result,
'dirlist': dir_result}
return jsonify(ret)
def filter_filelist(filelist, rules):
"""
Return a copy of the filelist where not-included and excluded files are removed according to the rule set.
"""
try:
filtered_filelist = copy.deepcopy(filelist)
for path, files in filelist['filelist'].items():
if not rules['dirs'].get(path):
# Probably got a directory from a wan filelist that does not exist on lan.
inf(f'filter filelist: ignoring directory "{path}" which is not found in rules'
)
else:
for filename in files:
if not filename_is_included(filename, rules['dirs'][path]):
del filtered_filelist['filelist'][path][filename]
except:
deb('http filelist returned unfiltered (bad rules?)')
return filtered_filelist
def popen_ignore_int(self, cmd):
'''Spawn a child command, ensuring it ignores SIGINT.
The returned subprocess.Popen object must be manually terminated.'''
cflags = 0
preexec = None
system = platform.system()
quoted = quote_sh_list(cmd)
if system == 'Windows':
cflags |= subprocess.CREATE_NEW_PROCESS_GROUP
elif system in {'Linux', 'Darwin'}:
preexec = os.setsid
if JUST_PRINT:
log.inf(quoted)
return _DebugDummyPopen()
log.dbg(quoted)
return subprocess.Popen(cmd, creationflags=cflags, preexec_fn=preexec)
def new_file(filename):
if not os.path.exists(filename):
deb(f'listener: changed file "{filename}" does not exist anymore?')
return
inf(f'listener: new/changed file "{FILUXE_LAN.log_path(filename)}"')
with Indent() as _:
if LAN_FILE_DELETER:
path = os.path.dirname(filename)
filestorage_path = os.path.relpath(path, FILUXE_LAN.root())
LAN_FILE_DELETER.enforce_max_files(filestorage_path,
rules=ACTIVE_RULES,
recursive=False)
if not os.path.exists(filename):
war(f'listener: new file "{FILUXE_LAN.log_path(filename)}" already deleted and will not be forwarded'
)
return
export_file(filename)
def do_run(self, args, ignored):
self.args = args # Avoid having to pass them around
log.dbg('args:', args, level=log.VERBOSE_EXTREME)
self._sanity_precheck()
self._setup_build_dir()
if is_zephyr_build(self.build_dir):
self._update_cache()
if self.args.cmake or self.args.cmake_opts:
self.run_cmake = True
else:
self.run_cmake = True
self._setup_source_dir()
self._sanity_check()
log.inf('source directory: {}'.format(self.source_dir), colorize=True)
log.inf('build directory: {}{}'.format(
self.build_dir, (' (created)' if self.created_build_dir else '')),
colorize=True)
if self.cmake_cache:
board = self.cmake_cache.get('CACHED_BOARD')
elif self.args.board:
board = self.args.board
else:
board = 'UNKNOWN' # shouldn't happen
log.inf('BOARD:', board, colorize=True)
self._run_cmake(self.args.cmake_opts)
self._sanity_check()
self._update_cache()
extra_args = ['--target', args.target] if args.target else []
cmake.run_build(self.build_dir, extra_args=extra_args)
def start(_, cfg):
global users
try:
file_root = cfg['wan_filestorage']
host = cfg['wan_host']
port = cfg['wan_port']
realm = 'WAN'
except:
file_root = cfg['lan_filestorage']
host = cfg['lan_host']
port = cfg['lan_port']
realm = 'LAN'
try:
for i in (0, 1):
if not os.path.exists(cfg['certificates'][i]):
die(f'specified certificate not found, "{cfg["certificates"][i]}"')
ssl_context = (cfg['certificates'][0], cfg['certificates'][1])
except:
ssl_context = False
if not os.path.exists(file_root):
try:
os.makedirs(file_root)
except:
die(f'unable to create file root {file_root}, please make it yourself and fix permissions',
ErrorCode.SERVER_ERROR)
app.config['fileroot'] = file_root
app.secret_key = os.urandom(50)
app.name = f'filuxe_server_{realm}'
try:
app.config['writekey'] = cfg['write_key']
except:
app.config['writekey'] = ''
try:
users = {cfg['username']: generate_password_hash(cfg['password'])}
inf('HTTP-AUTH enabled')
except:
inf('HTTP-AUTH disabled')
inf(f'filuxe {realm} server {filuxe_server_version} running at http{"s" if ssl_context else ""}://{host}:{port}')
inf(f'filestorage root "{file_root}"')
if ssl_context:
app.run(host=host, port=port, ssl_context=ssl_context)
else:
app.run(host=host, port=port)
return ErrorCode.OK
def __init__(self, cfg, lan=True, force=False):
self.certificate = False
protocol = 'http://'
self.force = force
try:
if lan:
self.domain = 'LAN'
try:
self.certificate = cfg['lan_certificate']
protocol = 'https://'
except:
pass
self.server = f'{protocol}{cfg["lan_host"]}:{cfg["lan_port"]}'
inf(f'filuxe LAN server is {self.server}')
self.file_root = cfg.get('lan_filestorage')
else:
self.domain = 'WAN'
try:
self.certificate = cfg['wan_certificate']
protocol = 'https://'
except:
pass
self.server = f'{protocol}{cfg["wan_host"]}:{cfg["wan_port"]}'
inf(f'filuxe WAN server is {self.server}')
try:
self.file_root = cfg['wan_filestorage']
except:
self.file_root = ''
except KeyError as e:
err(f'expected a {e} entry in the configuration file',
ErrorCode.MISSING_KEY)
try:
self.write_key = cfg['write_key']
except:
self.write_key = ''
def do_run(self, command, **kwargs):
bin_name = path.splitext(self.elf)[0] + path.extsep + 'bin'
cmd_convert = [self.espidf, '--chip', 'esp32', 'elf2image', self.elf]
cmd_flash = [
self.espidf, '--chip', 'esp32', '--port', self.device, '--baud',
self.baud, '--before', 'default_reset', '--after', 'hard_reset',
'write_flash', '-u', '--flash_mode', self.flash_mode,
'--flash_freq', self.flash_freq, '--flash_size', self.flash_size
]
if self.bootloader_bin:
cmd_flash.extend(['0x1000', self.bootloader_bin])
cmd_flash.extend(['0x8000', self.partition_table_bin])
cmd_flash.extend(['0x10000', bin_name])
else:
cmd_flash.extend(['0x1000', bin_name])
log.inf("Converting ELF to BIN")
self.check_call(cmd_convert)
log.inf("Flashing ESP32 on {} ({}bps)".format(self.device, self.baud))
self.check_call(cmd_flash)
def _dump_runner_opt_help(runner, cls):
# Construct and print the usage text
dummy_parser = argparse.ArgumentParser(prog='', add_help=False)
cls.add_parser(dummy_parser)
formatter = dummy_parser._get_formatter()
for group in dummy_parser._action_groups:
# Break the abstraction to filter out the 'flash', 'debug', etc.
# TODO: come up with something cleaner (may require changes
# in the runner core).
actions = group._group_actions
if len(actions) == 1 and actions[0].dest == 'command':
# This is the lone positional argument. Skip it.
continue
formatter.start_section('REMOVE ME')
formatter.add_text(group.description)
formatter.add_arguments(actions)
formatter.end_section()
# Get the runner help, with the "REMOVE ME" string gone
runner_help = '\n'.join(formatter.format_help().splitlines()[1:])
log.inf('{} options:'.format(runner), colorize=True)
log.inf(runner_help)
