def load_extension_specs(self):
if self.manifest is None:
# "None" means "extensions could not be determined".
# Leaving this an empty dict would mean "there are no
# extensions", which is different.
self.extensions = None
return
path_specs = extension_commands(manifest=self.manifest)
extension_names = set()
for path, specs in path_specs.items():
# Filter out attempts to shadow built-in commands as well as
# command names which are already used.
filtered = []
for spec in specs:
if spec.name in self.builtins:
log.wrn(f'ignoring project {spec.project.name} '
f'extension command "{spec.name}"; '
'this is a built in command')
continue
if spec.name in extension_names:
log.wrn(f'ignoring project {spec.project.name} '
f'extension command "{spec.name}"; '
f'command "{spec.name}" is '
'already defined as extension command')
continue
filtered.append(spec)
extension_names.add(spec.name)
self.extensions[spec.name] = spec
self.extension_groups[path] = filtered
def get_extension_commands():
extensions = extension_commands()
extension_names = set()
for path, specs in extensions.items():
# Filter out attempts to shadow built-in commands as well as
# commands which have names which are already used.
filtered = []
for spec in specs:
if spec.name in BUILTIN_COMMAND_NAMES:
log.wrn(
'ignoring project {} extension command {};'.format(
spec.project.name, spec.name),
'this is a built in command')
continue
if spec.name in extension_names:
log.wrn(
'ignoring project {} extension command "{}";'.format(
spec.project.name, spec.name),
'command "{}" already defined as extension command'.format(
spec.name))
continue
filtered.append(spec)
extension_names.add(spec.name)
extensions[path] = filtered
return extensions
def create(cls, cfg, args):
daparg = os.environ.get('PYOCD_DAPARG')
if daparg:
log.wrn('Setting PYOCD_DAPARG in the environment is',
'deprecated; use the --daparg option instead.')
if args.daparg is None:
log.dbg('Missing --daparg set to {} from environment'.format(
daparg),
level=log.VERBOSE_VERY)
args.daparg = daparg
build_conf = BuildConfiguration(cfg.build_dir)
flash_addr = cls.get_flash_address(args, build_conf)
return PyOcdBinaryRunner(cfg,
args.target,
flashtool=args.flashtool,
flash_addr=flash_addr,
flashtool_opts=args.flashtool_opt,
gdbserver=args.gdbserver,
gdb_port=args.gdb_port,
tui=args.tui,
board_id=args.board_id,
daparg=args.daparg,
frequency=args.frequency)
def do_run(self, args, unknown_args):
env = Manifest.from_file()
try:
kafl_path = env.get_projects(['kafl'])[0].abspath
except ValueError as e:
kafl_path = env.get_projects(['manifest'])[0].abspath
capstone_path = env.get_projects(['capstone'])[0].abspath
libxdc_path = env.get_projects(['libxdc'])[0].abspath
qemu_path = env.get_projects(['qemu'])[0].abspath
kafl_bin_path = os.path.join(kafl_path, 'kafl_fuzz.py')
if not os.path.exists(kafl_bin_path):
log.wrn("Could not find kAFL-Fuzzer in %s" % kafl_path)
kafl_bin_path = ""
qemu_bin_path = os.path.join(qemu_path,
'x86_64-softmmu/qemu-system-x86_64')
if not os.path.exists(qemu_bin_path):
log.wrn("Could not find kAFL Qemu binary in %s" % qemu_path)
qemu_bin_path = ""
# project executables
print("KAFL_BIN_PATH=%s" % kafl_bin_path)
print("KAFL_QEMU_PATH=%s" % qemu_bin_path)
# project libraries/includes
print("C_INCLUDE_PATH=%s/include:%s" % (capstone_path, libxdc_path))
print("LIBRARY_PATH=%s:%s" % (capstone_path, libxdc_path))
print("LD_LIBRARY_PATH=%s:%s" % (capstone_path, libxdc_path))
def do_run(self, args, extra_args):
env = Manifest.from_file()
for query in extra_args:
try:
prj = env.get_projects([query])
print(prj[0].abspath)
except ValueError as e:
# check if `manifest` is the kAFL repo..
if query != 'kafl':
log.err(
"Could not find %s in west projects. Try `west list`."
% query)
return
try:
# check if manifest repo is kAFL
kafl_path = env.get_projects(['manifest'])[0].abspath
if os.path.exists(kafl_path + '/kafl_fuzz.py'):
log.wrn(
"Returning `manifest` repo path for query `%s`.." %
query)
print(kafl_path)
except ValueError as e:
log.err(
"Could not find %s in west projects. Try `west list`."
% query)
except Exception as e:
log.err(str(e))
def print_loot(self, name, project, z_project, args):
# Print a list of out of tree outstanding patches in the given
# project.
#
# name: project name
# project: the west.manifest.Project instance in the NCS manifest
# z_project: the Project instance in the upstream manifest
name_path = _name_and_path(project)
# Get the upstream revision of the project. The zephyr project
# has to be treated as a special case.
if name == 'zephyr':
z_rev = self.zephyr_rev
else:
z_rev = z_project.revision
try:
nsha = project.sha(project.revision)
project.git('cat-file -e ' + nsha)
except subprocess.CalledProcessError:
log.wrn(f"{name_path}: can't get loot; please run "
f'"west update" (need revision {project.revision})')
return
try:
zsha = z_project.sha(z_rev)
z_project.git('cat-file -e ' + zsha)
except subprocess.CalledProcessError:
log.wrn(f"{name_path}: can't get loot; please fetch upstream URL "
f'{z_project.url} (need revision {z_project.revision})')
return
try:
analyzer = nwh.RepoAnalyzer(project, z_project, project.revision,
z_rev)
except nwh.InvalidRepositoryError as ire:
log.die(f"{name_path}: {str(ire)}")
try:
loot = analyzer.downstream_outstanding
except nwh.UnknownCommitsError as uce:
log.die(f'{name_path}: unknown commits: {str(uce)}')
if not loot and log.VERBOSE <= log.VERBOSE_NONE:
# Don't print output if there's no loot unless verbose
# mode is on.
return
log.banner(name_path)
log.inf(f'NCS commit: {nsha}, upstream commit: {zsha}')
log.inf('OOT patches: ' + (f'{len(loot)} total' if loot else 'none') +
(', output limited by --file' if args.files else ''))
for c in loot:
if args.files and not nwh.commit_affects_files(c, args.files):
log.dbg(f"skipping {c.oid}; it doesn't affect file filter",
level=log.VERBOSE_VERY)
continue
if args.sha_only:
log.inf(str(c.oid))
else:
log.inf(f'- {c.oid} {nwh.commit_shortlog(c)}')
def __init__(self, name, url):
if url.endswith('/'):
log.wrn('Remote', name, 'URL', url, 'ends with a slash ("/");',
'these are automatically appended by West')
self.name = name
self.url = url
def do_run(self, args, remainder):
self.args = args # Avoid having to pass them around
self.config_board = config_get('board', None)
log.dbg('args: {} remainder: {}'.format(args, remainder),
level=log.VERBOSE_EXTREME)
# Store legacy -s option locally
source_dir = self.args.source_dir
self._parse_remainder(remainder)
# Parse testcase.yaml or sample.yaml files for additional options.
if self.args.test_item:
self._parse_test_item()
if source_dir:
if self.args.source_dir:
log.die("source directory specified twice:({} and {})".format(
source_dir, self.args.source_dir))
self.args.source_dir = source_dir
log.dbg('source_dir: {} cmake_opts: {}'.format(self.args.source_dir,
self.args.cmake_opts),
level=log.VERBOSE_EXTREME)
self._sanity_precheck()
self._setup_build_dir()
if args.pristine is not None:
pristine = args.pristine
else:
# Load the pristine={auto, always, never} configuration value
pristine = config_get('pristine', 'never')
if pristine not in ['auto', 'always', 'never']:
log.wrn(
'treating unknown build.pristine value "{}" as "never"'.
format(pristine))
pristine = 'never'
self.auto_pristine = (pristine == 'auto')
log.dbg('pristine: {} auto_pristine: {}'.format(pristine,
self.auto_pristine),
level=log.VERBOSE_VERY)
if is_zephyr_build(self.build_dir):
if pristine == 'always':
self._run_pristine()
self.run_cmake = True
else:
self._update_cache()
if (self.args.cmake or self.args.cmake_opts or
self.args.cmake_only):
self.run_cmake = True
else:
self.run_cmake = True
self.source_dir = self._find_source_dir()
self._sanity_check()
board, origin = self._find_board()
self._run_cmake(board, origin, self.args.cmake_opts)
if args.cmake_only:
return
self._sanity_check()
self._update_cache()
self._run_build(args.target)
def do_run(self, args, user_args):
if args.exclude_west:
log.wrn('ignoring --exclude-west')
failed_rebases = []
for project in _projects(args, listed_must_be_cloned=False,
exclude_manifest=True):
_banner(project.format('updating {name_and_path}:'))
returncode = _update(project, args.rebase, args.keep_descendants)
if returncode:
failed_rebases.append(project)
log.err(project.format('{name_and_path} failed to rebase'))
if failed_rebases:
# Avoid printing this message if exactly one project
# was specified on the command line.
if len(args.projects) != 1:
log.err(('The following project{} failed to rebase; '
'see above for details: {}').format(
's' if len(failed_rebases) > 1 else '',
', '.join(p.format('{name_and_path}')
for p in failed_rebases)))
raise CommandError(1)
def do_run(self, args, ignored):
# Here we rely on main() having set up the west_parser
# attribute before calling run().
global BUILTIN_COMMANDS
if not args.command_name:
self.west_parser.print_help(top_level=True)
return
elif args.command_name == 'help':
self.parser.print_help()
return
elif args.command_name in BUILTIN_COMMANDS:
BUILTIN_COMMANDS[args.command_name].parser.print_help()
return
elif self.west_parser.west_extensions is not None:
extensions = self.west_parser.west_extensions.values()
for spec in itertools.chain(*extensions):
if spec.name == args.command_name:
# run_extension() does not return
run_extension(spec, self.topdir,
[args.command_name, '--help'], self.manifest)
log.wrn('unknown command "{}"'.format(args.command_name))
self.west_parser.print_help(top_level=True)
def load_manifest(self):
# Try to parse the manifest. We'll save it if that works, so
# it doesn't have to be re-parsed.
if not self.topdir:
return
try:
self.manifest = Manifest.from_file(topdir=self.topdir)
except (ManifestVersionError, MalformedManifest, MalformedConfig,
FileNotFoundError, ManifestImportFailed) as e:
# Defer exception handling to WestCommand.run(), which uses
# handle_builtin_manifest_load_err() to decide what to do.
#
# Make sure to update that function if you change the
# exceptions caught here. Unexpected exceptions should
# propagate up and fail fast.
#
# This might be OK, e.g. if we're running 'west config
# manifest.path foo' to fix the MalformedConfig error, but
# there's no way to know until we've parsed the command
# line arguments.
if isinstance(e, _ManifestImportDepth):
log.wrn('recursion depth exceeded during manifest resolution; '
'your manifest likely contains an import loop. '
'Run "west -v manifest --resolve" to debug.')
self.mle = e
def _post_checkout_help(project, branch, sha, is_ancestor):
# Print helpful information to the user about a project that
# might have just left a branch behind.
if branch is None:
# If there was no branch checked out, there are no
# additional diagnostics that need emitting.
return
rel = relpath(project.abspath)
if is_ancestor:
# If the branch we just left behind is a descendant of
# the new HEAD (e.g. if this is a topic branch the
# user is working on and the remote hasn't changed),
# print a message that makes it easy to get back,
# no matter where in the installation os.getcwd() is.
log.wrn(project.format(
'left behind {name} branch "{b}"; to switch '
'back to it (fast forward), use: git -C {rp} checkout {b}',
b=branch, rp=rel))
log.dbg('(To do this automatically in the future,',
'use "west update --keep-descendants".)')
else:
# Tell the user how they could rebase by hand, and
# point them at west update --rebase.
log.wrn(project.format(
'left behind {name} branch "{b}"; '
'to rebase onto the new HEAD: git -C {rp} rebase {sh} {b}',
b=branch, rp=rel, sh=sha))
log.dbg('(To do this automatically in the future,',
'use "west update --rebase".)')
def makeSPDX(cfg):
# report any odd configuration settings
if cfg.analyzeIncludes and not cfg.includeSDK:
log.wrn(f"config: requested to analyze includes but not to generate SDK SPDX document;")
log.wrn(f"config: will proceed but will discard detected includes for SDK header files")
# set up walker configuration
walkerCfg = WalkerConfig()
walkerCfg.namespacePrefix = cfg.namespacePrefix
walkerCfg.buildDir = cfg.buildDir
walkerCfg.analyzeIncludes = cfg.analyzeIncludes
walkerCfg.includeSDK = cfg.includeSDK
# make and run the walker
w = Walker(walkerCfg)
retval = w.makeDocuments()
if not retval:
log.err("SPDX walker failed; bailing")
return False
# set up scanner configuration
scannerCfg = ScannerConfig()
# scan each document from walker
if cfg.includeSDK:
scanDocument(scannerCfg, w.docSDK)
scanDocument(scannerCfg, w.docApp)
scanDocument(scannerCfg, w.docZephyr)
scanDocument(scannerCfg, w.docBuild)
# write each document, in this particular order so that the
# hashes for external references are calculated
# write SDK document, if we made one
if cfg.includeSDK:
retval = writeSPDX(os.path.join(cfg.spdxDir, "sdk.spdx"), w.docSDK)
if not retval:
log.err("SPDX writer failed for SDK document; bailing")
return False
# write app document
retval = writeSPDX(os.path.join(cfg.spdxDir, "app.spdx"), w.docApp)
if not retval:
log.err("SPDX writer failed for app document; bailing")
return False
# write zephyr document
writeSPDX(os.path.join(cfg.spdxDir, "zephyr.spdx"), w.docZephyr)
if not retval:
log.err("SPDX writer failed for zephyr document; bailing")
return False
# write build document
writeSPDX(os.path.join(cfg.spdxDir, "build.spdx"), w.docBuild)
if not retval:
log.err("SPDX writer failed for build document; bailing")
return False
return True
def do_run(self, args, unknown_args):
self.check_west_version()
self.setup_upstream_downstream(args)
log.inf('Comparing nrf/west.yml with zephyr/west.yml at revision {}{}'.
format(self.zephyr_rev,
(', sha: ' + self.zephyr_sha
if self.zephyr_rev != self.zephyr_sha else '')))
log.inf()
present_blacklisted = []
present_allowed = []
missing_blacklisted = []
missing_allowed = []
for zn, zp in self.z_pmap.items():
nn = self.to_ncs_name(zp)
present = nn in self.ncs_pmap
blacklisted = PurePath(zp.path) in _PROJECT_BLACKLIST
if present:
if blacklisted:
present_blacklisted.append(zp)
else:
present_allowed.append(zp)
else:
if blacklisted:
missing_blacklisted.append(zp)
else:
missing_allowed.append(zp)
def print_lst(projects):
for p in projects:
log.small_banner(p.format('{name_and_path}'))
if missing_blacklisted and log.VERBOSE >= log.VERBOSE_NORMAL:
log.banner('blacklisted but missing in NCS (these are all OK):')
print_lst(missing_blacklisted)
if present_blacklisted:
log.banner('blacklisted upstream projects that are in NCS:')
log.wrn('these names should be removed from nrf/west.yml')
print_lst(present_blacklisted)
if missing_allowed:
log.banner('non-blacklisted upstream projects missing in NCS:')
log.wrn('these should be blacklisted or added to nrf/west.yml')
for p in missing_allowed:
log.small_banner(p.format('{name_and_path}'))
log.inf(p.format('upstream revision: {revision}'))
log.inf(p.format('upstream URL: {url}'))
if present_allowed:
log.banner('projects present in NCS:')
for zp in present_allowed:
# Do some extra checking on unmerged commits.
self.allowed_project(zp)
def print_loot(self, name, project, z_project, args):
# Print a list of out of tree outstanding patches in the given
# project.
#
# name: project name
# project: the west.manifest.Project instance in the NCS manifest
# z_project: the Project instance in the upstream manifest
msg = project.format('{name_and_path} outstanding downstream patches:')
# Get the upstream revision of the project. The zephyr project
# has to be treated as a special case.
if name == 'zephyr':
z_rev = self.zephyr_rev
msg += ' NOTE: {} *must* be up to date'.format(z_rev)
else:
z_rev = z_project.revision
log.banner(msg)
try:
nsha = project.sha(project.revision)
project.git('cat-file -e ' + nsha)
except subprocess.CalledProcessError:
log.wrn(
"can't get loot; please run \"west update {}\"".format(
project.name),
'(need revision {})'.format(project.revision))
return
try:
zsha = z_project.sha(z_project.revision)
z_project.git('cat-file -e ' + zsha)
except subprocess.CalledProcessError:
log.wrn("can't get loot; please fetch upstream URL", z_project.url,
'(need revision {})'.format(z_project.revision))
return
try:
analyzer = nwh.RepoAnalyzer(project, z_project, project.revision,
z_rev)
except nwh.InvalidRepositoryError as ire:
log.die(str(ire))
try:
for c in analyzer.downstream_outstanding:
if args.files and not nwh.commit_affects_files(c, args.files):
log.dbg(
'skipping {}; it does not affect file filter'.format(
c.oid),
level=log.VERBOSE_VERY)
continue
if args.sha_only:
log.inf(str(c.oid))
else:
log.inf('- {} {}'.format(c.oid, nwh.commit_shortlog(c)))
except nwh.UnknownCommitsError as uce:
log.die('unknown commits:', str(uce))
def clean_up(path):
try:
run_cmake(['-P', str(path / 'pristine.cmake')],
capture_output=True)
except CalledProcessError:
# Do our best to clean up even though CMake failed.
log.wrn(f'Failed to make {path} pristine; '
'removing known generated files...')
for subpath in ['CMakeCache.txt', 'CMakeFiles', 'build.ninja',
'cmake_install.cmake', 'rules.ninja']:
remove_if_exists(Path(path) / subpath)
def set_zephyr_base(args):
'''Ensure ZEPHYR_BASE is set, emitting warnings if that's not
possible, or if the user is pointing it somewhere different than
what the manifest expects.'''
zb_env = os.environ.get('ZEPHYR_BASE')
if args.zephyr_base:
# The command line --zephyr-base takes precedence over
# everything else.
zb = os.path.abspath(args.zephyr_base)
zb_origin = 'command line'
else:
# If the user doesn't specify it concretely, use the project
# with path 'zephyr' if that exists, or the ZEPHYR_BASE value
# in the calling environment.
#
# At some point, we need a more flexible way to set environment
# variables based on manifest contents, but this is good enough
# to get started with and to ask for wider testing.
try:
manifest = Manifest.from_file()
except MalformedConfig as e:
log.die('Parsing of manifest file failed during command',
args.command, ':', *e.args)
for project in manifest.projects:
if project.path == 'zephyr':
zb = project.abspath
zb_origin = 'manifest file {}'.format(manifest.path)
break
else:
if zb_env is None:
log.wrn('no --zephyr-base given, ZEPHYR_BASE is unset,',
'and no manifest project has path "zephyr"')
zb = None
zb_origin = None
else:
zb = zb_env
zb_origin = 'environment'
if zb_env and os.path.abspath(zb) != os.path.abspath(zb_env):
# The environment ZEPHYR_BASE takes precedence over either the
# command line or the manifest, but in normal multi-repo
# operation we shouldn't expect to need to set ZEPHYR_BASE to
# point to some random place. In practice, this is probably
# happening because zephyr-env.sh/cmd was run in some other
# zephyr installation, and the user forgot about that.
log.wrn('ZEPHYR_BASE={}'.format(zb_env),
'in the calling environment, but has been set to', zb,
'instead by the', zb_origin)
os.environ['ZEPHYR_BASE'] = zb
log.dbg('ZEPHYR_BASE={} (origin: {})'.format(zb, zb_origin))
def fetch_strategy(self, args):
cfg = config.get('update', 'fetch', fallback=None)
if cfg is not None and cfg not in ('always', 'smart'):
log.wrn(f'ignoring invalid config update.fetch={cfg}; '
'choices: always, smart')
cfg = None
if args.fetch_strategy:
return args.fetch_strategy
elif cfg:
return cfg
else:
return 'smart'
def do_run(self, args, user_args):
self.args = args
if args.exclude_west:
log.wrn('ignoring --exclude-west')
# We can't blindly call self._projects() here: manifests with
# imports are limited to plain 'west update', and cannot use
# 'west update PROJECT [...]'.
self.fs = self.fetch_strategy(args)
if not args.projects:
self.update_all(args)
else:
self.update_some(args)
def command_execute(*cmd_args: 'tuple[str|Path]',
cwd: 'str|Path|None' = None,
return_path: bool = False,
allow_stderr: bool = False) -> 'Path|str':
'''Execute subprocess wrapper that handles errors and output redirections.'''
cmd_args = tuple(str(x) for x in cmd_args)
if cwd is not None:
cwd = str(cwd)
with NamedTemporaryFile(delete=(not return_path), mode='w+') as out_file, \
NamedTemporaryFile(mode='w+') as err_file:
try:
t = dbg_time(
f'Starting {cmd_args} in {cwd or "current directory"}',
level=log.VERBOSE_VERY)
cp = subprocess.run(cmd_args,
stdout=out_file,
stderr=err_file,
cwd=cwd)
log.dbg(f'Subprocess done in {t}s', level=log.VERBOSE_VERY)
except Exception as e:
log.err(f'Running command "{cmd_args[0]}" failed!')
log.err(f'Arguments: { json.dumps(cmd_args) }, cwd: "{cwd}"')
log.err(f'Details: {e}')
raise SbomException('Command execution error.') from e
out_file.seek(0)
err_file.seek(0)
err = err_file.read()
if len(err.strip()) > 0:
if allow_stderr:
log.wrn(f'Command "{cmd_args[0]}" reported errors:\n{err}')
else:
log.err(f'Command "{cmd_args[0]}" reported errors:\n{err}')
if cp.returncode == 0:
log.err(
f'Arguments: { json.dumps(cmd_args) }, cwd: "{cwd}"')
raise SbomException('Command execution error.')
err_file.close()
if cp.returncode != 0:
log.err(
f'Command "{cmd_args[0]}" exited with error code {cp.returncode}'
)
log.err(f'Arguments: { json.dumps(cmd_args) }, cwd: "{cwd}"')
raise SbomException('Command execution error.')
if return_path:
return Path(out_file.name)
else:
return out_file.read()
def _run_cmake(self, board, origin, cmake_opts):
if board is None and config_getboolean('board_warn', True):
log.wrn('This looks like a fresh build and BOARD is unknown;',
"so it probably won't work. To fix, use",
'--board=<your-board>.')
log.inf('Note: to silence the above message, run',
"'west config build.board_warn false'")
if not self.run_cmake:
return
_banner('generating a build system')
if board is not None and origin != 'CMakeCache.txt':
cmake_opts = ['-DBOARD={}'.format(board)]
else:
cmake_opts = []
if self.args.cmake_opts:
cmake_opts.extend(self.args.cmake_opts)
user_args = config_get('cmake-args', None)
if user_args:
cmake_opts.extend(shlex.split(user_args))
config_sysbuild = config_getboolean('sysbuild', False)
if self.args.sysbuild or (config_sysbuild
and not self.args.no_sysbuild):
cmake_opts.extend([
'-S{}'.format(SYSBUILD_PROJ_DIR),
'-DAPP_DIR:PATH={}'.format(self.source_dir)
])
else:
# self.args.no_sysbuild == True or config sysbuild False
cmake_opts.extend(['-S{}'.format(self.source_dir)])
# Invoke CMake from the current working directory using the
# -S and -B options (officially introduced in CMake 3.13.0).
# This is important because users expect invocations like this
# to Just Work:
#
# west build -- -DOVERLAY_CONFIG=relative-path.conf
final_cmake_args = [
'-DWEST_PYTHON={}'.format(sys.executable),
'-B{}'.format(self.build_dir),
'-G{}'.format(config_get('generator', DEFAULT_CMAKE_GENERATOR))
]
if cmake_opts:
final_cmake_args.extend(cmake_opts)
run_cmake(final_cmake_args, dry_run=self.args.dry_run)
def do_run(self, args, user_args):
if args.exclude_west:
log.wrn('ignoring --exclude-west')
fs = self._fetch_strategy(args)
failed = []
for project in self._projects(args.projects):
if isinstance(project, ManifestProject):
continue
log.banner(project.format('updating {name_and_path}:'))
try:
_update(project, fs, args.rebase, args.keep_descendants)
except subprocess.CalledProcessError:
failed.append(project)
self._handle_failed(args, failed)
def scanDocument(cfg, doc):
"""
Scan for licenses and calculate hashes for all Files and Packages
in this Document.
Arguments:
- cfg: ScannerConfig
- doc: Document
"""
for pkg in doc.pkgs.values():
log.inf(
f"scanning files in package {pkg.cfg.name} in document {doc.cfg.name}"
)
# first, gather File data for this package
for f in pkg.files.values():
# set relpath based on package's relativeBaseDir
f.relpath = os.path.relpath(f.abspath, pkg.cfg.relativeBaseDir)
# get hashes for file
hashes = getHashes(f.abspath)
if not hashes:
log.wrn("unable to get hashes for file {f.abspath}; skipping")
continue
hSHA1, hSHA256, hMD5 = hashes
f.sha1 = hSHA1
if cfg.doSHA256:
f.sha256 = hSHA256
if cfg.doMD5:
f.md5 = hMD5
# get licenses for file
expression = getExpressionData(f.abspath, cfg.numLinesScanned)
if expression:
if cfg.shouldConcludeFileLicenses:
f.concludedLicense = expression
f.licenseInfoInFile = splitExpression(expression)
# check if any custom license IDs should be flagged for document
for lic in f.licenseInfoInFile:
checkLicenseValid(lic, doc)
# now, assemble the Package data
licsConcluded, licsFromFiles = getPackageLicenses(pkg)
if cfg.shouldConcludePackageLicense:
pkg.concludedLicense = normalizeExpression(licsConcluded)
pkg.licenseInfoFromFiles = licsFromFiles
pkg.verificationCode = calculateVerificationCode(pkg)
