def run_scan_plain(
options,
cwd=None,
test_mode=True,
expected_rc=0,
env=None,
retry=True,
):
"""
Run a scan as a plain subprocess. Return rc, stdout, stderr.
"""
from commoncode.command import execute
options = add_windows_extra_timeout(options)
if test_mode and '--test-mode' not in options:
options.append('--test-mode')
if not env:
env = dict(os.environ)
scmd = 'scancode'
scan_cmd = os.path.join(scancode_root_dir, scmd)
rc, stdout, stderr = execute(
cmd_loc=scan_cmd,
args=options,
cwd=cwd,
env=env,
)
if retry and rc != expected_rc:
# wait and rerun in verbose mode to get more in the output
time.sleep(1)
if '--verbose' not in options:
options.append('--verbose')
result = rc, stdout, stderr = execute(
cmd_loc=scan_cmd,
args=options,
cwd=cwd,
env=env,
)
if rc != expected_rc:
opts = get_opts(options)
error = f'''
Failure to run:
rc: {rc}
scancode {opts}
stdout:
{stdout}
stderr:
{stderr}
''' % locals()
assert rc == expected_rc, error
return rc, stdout, stderr
def parse(self):
"""
Parse readelf sections to populates the Elf object for an elf location.
"""
readelf_args = ['--wide']
readelf_args.extend(self.readelf_options)
readelf_args.append(self.elf_location)
readelf_command = 'readelf'
rc, out, err = command.execute(cmd=readelf_command,
args=readelf_args,
root_dir=bin_dir,
to_files=True)
if rc != 0:
raise Exception(open(err).read() + '\n' + open(out).read())
# loop through each line passing control to a handler as needed
with open(out, 'rb') as elf_lines:
for line in elf_lines:
# get the handler for this line
line = line.strip()
if line:
handler = self.get_handler(line)
if handler:
handler(self, elf_lines)
def demangle_chunk(symbols):
"""
Return a set of demangled Elf symbol names using binutils
c++filt. The symbols are filtered for eventual known junk.
"""
if not symbols:
return []
cppfilt_command = 'c++filt'
args = ['--no-strip-underscores', '--no-verbose', '--no-params'] + symbols
rc, out, err = command.execute(cppfilt_command,
args,
root_dir=bin_dir,
to_files=True)
if rc != 0:
raise Exception(open(err).read())
demangled = set()
with open(out, 'rb') as names:
for name in names:
# ignore junk injected by the compiler
isjunk = False
for junk in demangled_junk:
if name.startswith(junk):
isjunk = True
break
if isjunk:
continue
# do not keep params for CPP functions, just the function
if '(' in name:
name = name.split('(')[0]
demangled.add(name.strip())
return list(demangled)
def run_scan_plain(options, cwd=None):
"""
Run a scan as a plain subprocess. Return rc, stdout, stderr.
"""
import scancode
from commoncode.command import execute
scan_cmd = os.path.join(scancode.root_dir, 'scancode')
return execute(scan_cmd, options, cwd=cwd)
def test_execute_(self):
python = sys.executable
rc, stdout, stderr = command.execute(python, ['-c', 'print("foobar")'])
assert stderr == ''
assert stdout == 'foobar'
assert rc == 0
# do not throw exception
stdout.encode('ascii')
def run_scan_plain(options, cwd=None):
"""
Run a scan as a plain subprocess. Return rc, stdout, stderr.
"""
import scancode
scmd = b'scancode' if on_linux else 'scancode'
from commoncode.command import execute
scan_cmd = os.path.join(scancode.root_dir, scmd)
return execute(scan_cmd, options, cwd=cwd)
def installed_rpms(image_id):
"""
Return a list of installed RPMs in an installed Docker image id.
"""
# lines format: one rpm as NVRA per line
rc, stdout, _stderr = command.execute(
'docker', ['run', image_id, 'rpm', '--query', '--all'])
# we just skip if things do not run OK.
return stdout.splitlines(False) if rc >= 0 else []
def test_execute_can_handle_non_ascii_output(self):
# Popen returns a *binary* string with non-ascii chars: skips these
python = sys.executable
rc, stdout, stderr = command.execute(
python, ['-c', 'print("non ascii: été just passed it !")'])
assert stderr == ''
assert stdout == 'non ascii: ete just passed it !'
assert rc == 0
# do not throw exception
stdout.encode('ascii')
def test_execute_non_ascii_output(self):
# Popen returns a *binary* string with non-ascii chars: skips these
rc, stdout, stderr = command.execute(
b'python', ['-c', "print b'non ascii: \\xe4 just passed it !'"])
assert rc == 0
assert stderr == b''
# converting to Unicode could cause an "ordinal not in range..."
# exception
assert stdout == b'non ascii: just passed it !'
unicode(stdout)
def call_nm(elffile):
"""
Call nm and returns the returncode, and the filepaths containing the
stdout and stderr.
"""
logger.debug('Executing nm command on %(elffile)r' % locals())
nm_command = 'nm-new'
return command.execute(cmd=nm_command,
args=['-al', elffile],
root_dir=bin_dir,
to_files=True)
def test_execute_non_ascii_output(self):
# Popen returns a *binary* string with non-ascii chars: skips these
rc, stdout, stderr = command.execute(
'python', ['-c', "print 'non ascii: \\xe4 just passed it !'"]
)
assert rc == 0
assert stderr == ''
# converting to Unicode could cause an "ordinal not in range..."
# exception
assert stdout == 'non ascii: just passed it !'
unicode(stdout)
def call_nm(elffile):
"""
Call nm and returns the returncode, and the filepaths containing the
stdout and stderr.
"""
logger.debug('Executing nm command on %(elffile)r' % locals())
nm_command = get_location(SCANCODE_BINUTILS_NM_EXE)
return command.execute(
cmd_loc=nm_command,
args=['-al', elffile],
to_files=True,
)
def test_can_process_scan_from_json_scan(self):
import scancode
from commoncode.command import execute
test_dir = self.get_test_loc('livescan/scan')
json_file = self.get_temp_file('json')
scan_cmd = os.path.join(scancode.root_dir, 'scancode')
rc, _stdout, _stderr = execute(scan_cmd,
['-clip', '--email', '--url', '--strip-root', '--format', 'json', test_dir, json_file])
assert rc == 0
result_file = self.get_temp_file('.csv')
with open(result_file, 'wb') as rf:
json2csv.json_scan_to_csv(json_file, rf)
expected_file = self.get_test_loc('livescan/expected.csv')
check_csvs(result_file, expected_file)
def list_entries(location, arch_type='*'):
"""
List entries from a 7zip-supported archive file at location.
Yield Entry tuples.
Use the -t* 7z cli type option or the provided arch_type 7z type (can be
None).
"""
assert location
abs_location = os.path.abspath(os.path.expanduser(location))
# 7z arguments
listing = 'l'
# NB: we use t* to ensure that all archive types are honored
if not arch_type:
arch_type = ''
else:
arch_type = '-t' + arch_type
# pass an empty password so that extraction with passwords WILL fail
password = '******'
tech_info = '-slt'
output_as_utf = ''
if on_windows:
output_as_utf = '-sccUTF-8'
# NB: we force running in the GMT timezone, because 7z is unable to set
# the TZ correctly when the archive does not contain TZ info. This does
# not work on Windows, because 7z is not using the TZ env var there.
timezone = os.environ.update({'TZ': 'GMT'})
args = [
listing, tech_info, arch_type, output_as_utf, password, abs_location
]
rc, stdout, _stderr = command.execute(cmd='7z',
args=args,
env=timezone,
root_dir=root_dir,
to_files=True)
if rc != 0:
_error = get_7z_errors(stdout) or 'No error returned'
# still try to get the listing?
# print(Exception(error))
pass
# the listing was produced as UTF on windows to avoid damaging binary
# paths in console outputs
utf = bool(output_as_utf)
return parse_7z_listing(stdout, utf)
def list_entries(location, arch_type='*'):
"""
List entries from a 7zip-supported archive file at location.
Yield Entry tuples.
Use the -t* 7z cli type option or the provided arch_type 7z type (can be
None).
"""
assert location
abs_location = os.path.abspath(os.path.expanduser(location))
# 7z arguments
listing = 'l'
# NB: we use t* to ensure that all archive types are honored
if not arch_type:
arch_type = ''
else:
arch_type = '-t' + arch_type
# pass an empty password so that extraction with passwords WILL fail
password = '******'
tech_info = '-slt'
output_as_utf = ''
if on_windows:
output_as_utf = '-sccUTF-8'
# NB: we force running in the GMT timezone, because 7z is unable to set
# the TZ correctly when the archive does not contain TZ info. This does
# not work on Windows, because 7z is not using the TZ env var there.
timezone = os.environ.update({'TZ': 'GMT'})
args = [listing, tech_info, arch_type, output_as_utf, password, abs_location]
rc, stdout, _stderr = command.execute(cmd='7z',
args=args,
env=timezone,
root_dir=root_dir,
to_files=True)
if rc != 0:
_error = get_7z_errors(stdout) or 'No error returned'
# still try to get the listing?
# print(Exception(error))
pass
# the listing was produced as UTF on windows to avoid damaging binary
# paths in console outputs
utf = bool(output_as_utf)
return parse_7z_listing(stdout, utf)
def test_can_process_scan_from_json_scan(self):
import scancode
from commoncode.command import execute
test_dir = self.get_test_loc('livescan/scan')
json_file = self.get_temp_file('json')
scan_cmd = os.path.join(scancode.root_dir, 'scancode')
rc, _stdout, _stderr = execute(scan_cmd, [
'-clip', '--email', '--url', '--strip-root', '--format', 'json',
test_dir, json_file
])
assert rc == 0
result_file = self.get_temp_file('.csv')
with open(result_file, 'wb') as rf:
json2csv.json_scan_to_csv(json_file, rf)
expected_file = self.get_test_loc('livescan/expected.csv')
check_csvs(result_file, expected_file)
def installed_images(image_id=None):
"""
Return a list of locally installed Docker images as a three tuple of (name,
tag, image id).
"""
# REPOSITORY TAG IMAGE ID CREATED VIRTUAL SIZE
# mine/my-cloud 1.4.0.45 9b63d7aa7390 10 weeks ago 296.9 MB
rc, stdout, _stderr = command.execute('docker', ['images', '--no-trunc'])
# we just skip if things do not run OK.
if rc >= 0:
# skip the first header line
for line in stdout.splitlines(False)[1:]:
# split on spaces and keep the first three elements
name, tag, imid = line.split()[:3]
imid = get_real_id(imid)
logger.info('installed_images: %(name)s, %(tag)s, %(imid)s' %
locals())
if not image_id or (image_id and imid.startswith(image_id)):
yield name, tag, imid
def _parseinfo(self):
"""
Parse dwarfdump info section of an elf file.
"""
dwarfdump_command = 'dwarfdump2'
rc, out, err = command.execute(cmd=dwarfdump_command,
args=['-i', self.elf_location],
root_dir=bin_dir,
to_files=True)
if rc != 0:
raise Exception(open(err).read())
# loop through each returned line passing control to a handler
with open(out, 'rb') as lines:
for line in lines:
line = line.strip()
if DCOMP_UNIT_START_RE().match(line):
dwarfinfo = DwarfInfo()
dwarfinfo.parse(self, lines)
def installed_image_history(image_id):
"""
Return a list of ordered installed Docker layers ID given an image id as a
tuple of (layer id, comment). The history is from latest to oldest.
"""
# lines format
# IMAGE CREATED CREATED BY SIZE COMMENT
# 9b63d75a7390 10 weeks ago /bin/sh -c #(nop) CMD [/bin/sh -c /startConfd 0 B
rc, stdout, _stderr = command.execute('docker',
['history', '--no-trunc', image_id])
# we just skip if things do not run OK.
if rc >= 0:
# skip the first header line
for line in stdout.splitlines(False)[1:]:
# progressively partition on two spaces
layer_id, _, right = line.partition(' ')
_, _, right = right.strip().partition(' ')
layer_command, _, right = right.strip().partition(' ')
_size, _, right = right.strip().partition(' ')
_comment = right.strip()
yield get_real_id(layer_id), layer_command
def get_msi_info(location):
"""
Run the command `msiinfo suminfo` on the file at `location` and return the
results in a dictionary
This function requires `msiinfo` to be installed on the system, either by
installing the `packagedcode-msiinfo` plugin or by installing `msitools`
through a package manager.
"""
rc, stdout, stderr = execute(
cmd_loc=get_msiinfo_bin_location(),
args=[
'suminfo',
location,
],
)
if stderr:
error_message = f'Error encountered when reading MSI information from {location}: '
error_message = error_message + stderr
raise MsiinfoException(error_message)
return parse_msiinfo_suminfo_output(stdout)
def _collect_and_parse_tags(self):
ctags_args = ['--fields=K', '--c-kinds=fp', '-f', '-', self.sourcefile]
ctags_temp_dir = fileutils.get_temp_dir(base_dir='ctags')
envt = {'TMPDIR': ctags_temp_dir}
try:
rc, stdo, err = command.execute('ctags',
ctags_args,
env=envt,
root_dir=bin_dir,
to_files=True)
if rc != 0:
raise Exception(open(err).read())
with open(stdo, 'rb') as lines:
for line in lines:
if 'cannot open temporary file' in line:
raise Exception('ctags: cannot open temporary file '
': Permission denied')
if line.startswith('!'):
continue
line = line.strip()
if not line:
continue
splitted = line.split('\t')
if (line.endswith('function\tfile:')
or line.endswith('prototype\tfile:')):
self.local_functions.append(splitted[0])
elif (line.endswith('function')
or line.endswith('prototype')):
self.global_functions.append(splitted[0])
finally:
fileutils.delete(ctags_temp_dir)
def extract(location, target_dir, arch_type='*'):
"""
Extract all files from a 7zip-supported archive file at location in the
target_dir directory. Return a list of warning messages.
Raise exception on errors.
`arch_type` is the type of 7zip archive passed to the -t 7zip option. Can be
None.
"""
assert location
assert target_dir
abs_location = os.path.abspath(os.path.expanduser(location))
abs_target_dir = os.path.abspath(os.path.expanduser(target_dir))
# note: there are some issues with the extraction of debian .deb ar files
# see sevenzip bug http://sourceforge.net/p/sevenzip/bugs/1472/
# 7z arguments
extract = 'x'
yes_to_all = '-y'
# NB: we use t* to ensure that all archive types are honored
if not arch_type:
arch_type = ''
else:
arch_type = '-t' + arch_type
# pass an empty password so that extraction with passwords WILL fail
password = '******'
# renaming may not behave the same way on all OSes in particular Mac and Windows
auto_rename_dupe_names = '-aou'
# These things do not work well with p7zip for now:
# - ensure that we treat the FS as case insensitive even if it is
# this ensure we have consistent names across OSes
# case_insensitive = '-ssc-'
# - force any console output to be UTF-8 encoded
# TODO: add this may be for a UTF output on Windows only
# output_as_utf = '-sccUTF-8'
# working_tmp_dir = '-w<path>'
# NB: we force running in the GMT timezone, because 7z is unable to set
# the TZ correctly when the archive does not contain TZ info. This does
# not work on Windows, because 7z is not using the TZ env var there.
timezone = os.environ.update({'TZ': 'GMT'})
# Note: 7z does extract in the current directory so we cwd to the target dir first
args = [extract, yes_to_all, auto_rename_dupe_names,
arch_type, password, abs_location]
rc, stdout, _stderr = command.execute(
cmd='7z',
args=args,
cwd=abs_target_dir,
env=timezone,
root_dir=root_dir
)
if rc != 0:
error = get_7z_errors(stdout) or UNKNOWN_ERROR
raise ExtractErrorFailedToExtract(error)
extractcode.remove_backslashes_and_dotdots(abs_target_dir)
return get_7z_warnings(stdout)