def run_scan_plain( options, cwd=None, test_mode=True, expected_rc=0, env=None, retry=True, ): """ Run a scan as a plain subprocess. Return rc, stdout, stderr. """ from commoncode.command import execute options = add_windows_extra_timeout(options) if test_mode and '--test-mode' not in options: options.append('--test-mode') if not env: env = dict(os.environ) scmd = 'scancode' scan_cmd = os.path.join(scancode_root_dir, scmd) rc, stdout, stderr = execute( cmd_loc=scan_cmd, args=options, cwd=cwd, env=env, ) if retry and rc != expected_rc: # wait and rerun in verbose mode to get more in the output time.sleep(1) if '--verbose' not in options: options.append('--verbose') result = rc, stdout, stderr = execute( cmd_loc=scan_cmd, args=options, cwd=cwd, env=env, ) if rc != expected_rc: opts = get_opts(options) error = f''' Failure to run: rc: {rc} scancode {opts} stdout: {stdout} stderr: {stderr} ''' % locals() assert rc == expected_rc, error return rc, stdout, stderr
def parse(self): """ Parse readelf sections to populates the Elf object for an elf location. """ readelf_args = ['--wide'] readelf_args.extend(self.readelf_options) readelf_args.append(self.elf_location) readelf_command = 'readelf' rc, out, err = command.execute(cmd=readelf_command, args=readelf_args, root_dir=bin_dir, to_files=True) if rc != 0: raise Exception(open(err).read() + '\n' + open(out).read()) # loop through each line passing control to a handler as needed with open(out, 'rb') as elf_lines: for line in elf_lines: # get the handler for this line line = line.strip() if line: handler = self.get_handler(line) if handler: handler(self, elf_lines)
def demangle_chunk(symbols): """ Return a set of demangled Elf symbol names using binutils c++filt. The symbols are filtered for eventual known junk. """ if not symbols: return [] cppfilt_command = 'c++filt' args = ['--no-strip-underscores', '--no-verbose', '--no-params'] + symbols rc, out, err = command.execute(cppfilt_command, args, root_dir=bin_dir, to_files=True) if rc != 0: raise Exception(open(err).read()) demangled = set() with open(out, 'rb') as names: for name in names: # ignore junk injected by the compiler isjunk = False for junk in demangled_junk: if name.startswith(junk): isjunk = True break if isjunk: continue # do not keep params for CPP functions, just the function if '(' in name: name = name.split('(')[0] demangled.add(name.strip()) return list(demangled)
def run_scan_plain(options, cwd=None): """ Run a scan as a plain subprocess. Return rc, stdout, stderr. """ import scancode from commoncode.command import execute scan_cmd = os.path.join(scancode.root_dir, 'scancode') return execute(scan_cmd, options, cwd=cwd)
def test_execute_(self): python = sys.executable rc, stdout, stderr = command.execute(python, ['-c', 'print("foobar")']) assert stderr == '' assert stdout == 'foobar' assert rc == 0 # do not throw exception stdout.encode('ascii')
def run_scan_plain(options, cwd=None): """ Run a scan as a plain subprocess. Return rc, stdout, stderr. """ import scancode scmd = b'scancode' if on_linux else 'scancode' from commoncode.command import execute scan_cmd = os.path.join(scancode.root_dir, scmd) return execute(scan_cmd, options, cwd=cwd)
def installed_rpms(image_id): """ Return a list of installed RPMs in an installed Docker image id. """ # lines format: one rpm as NVRA per line rc, stdout, _stderr = command.execute( 'docker', ['run', image_id, 'rpm', '--query', '--all']) # we just skip if things do not run OK. return stdout.splitlines(False) if rc >= 0 else []
def test_execute_can_handle_non_ascii_output(self): # Popen returns a *binary* string with non-ascii chars: skips these python = sys.executable rc, stdout, stderr = command.execute( python, ['-c', 'print("non ascii: été just passed it !")']) assert stderr == '' assert stdout == 'non ascii: ete just passed it !' assert rc == 0 # do not throw exception stdout.encode('ascii')
def test_execute_non_ascii_output(self): # Popen returns a *binary* string with non-ascii chars: skips these rc, stdout, stderr = command.execute( b'python', ['-c', "print b'non ascii: \\xe4 just passed it !'"]) assert rc == 0 assert stderr == b'' # converting to Unicode could cause an "ordinal not in range..." # exception assert stdout == b'non ascii: just passed it !' unicode(stdout)
def call_nm(elffile): """ Call nm and returns the returncode, and the filepaths containing the stdout and stderr. """ logger.debug('Executing nm command on %(elffile)r' % locals()) nm_command = 'nm-new' return command.execute(cmd=nm_command, args=['-al', elffile], root_dir=bin_dir, to_files=True)
def test_execute_non_ascii_output(self): # Popen returns a *binary* string with non-ascii chars: skips these rc, stdout, stderr = command.execute( 'python', ['-c', "print 'non ascii: \\xe4 just passed it !'"] ) assert rc == 0 assert stderr == '' # converting to Unicode could cause an "ordinal not in range..." # exception assert stdout == 'non ascii: just passed it !' unicode(stdout)
def call_nm(elffile): """ Call nm and returns the returncode, and the filepaths containing the stdout and stderr. """ logger.debug('Executing nm command on %(elffile)r' % locals()) nm_command = get_location(SCANCODE_BINUTILS_NM_EXE) return command.execute( cmd_loc=nm_command, args=['-al', elffile], to_files=True, )
def test_can_process_scan_from_json_scan(self): import scancode from commoncode.command import execute test_dir = self.get_test_loc('livescan/scan') json_file = self.get_temp_file('json') scan_cmd = os.path.join(scancode.root_dir, 'scancode') rc, _stdout, _stderr = execute(scan_cmd, ['-clip', '--email', '--url', '--strip-root', '--format', 'json', test_dir, json_file]) assert rc == 0 result_file = self.get_temp_file('.csv') with open(result_file, 'wb') as rf: json2csv.json_scan_to_csv(json_file, rf) expected_file = self.get_test_loc('livescan/expected.csv') check_csvs(result_file, expected_file)
def list_entries(location, arch_type='*'): """ List entries from a 7zip-supported archive file at location. Yield Entry tuples. Use the -t* 7z cli type option or the provided arch_type 7z type (can be None). """ assert location abs_location = os.path.abspath(os.path.expanduser(location)) # 7z arguments listing = 'l' # NB: we use t* to ensure that all archive types are honored if not arch_type: arch_type = '' else: arch_type = '-t' + arch_type # pass an empty password so that extraction with passwords WILL fail password = '******' tech_info = '-slt' output_as_utf = '' if on_windows: output_as_utf = '-sccUTF-8' # NB: we force running in the GMT timezone, because 7z is unable to set # the TZ correctly when the archive does not contain TZ info. This does # not work on Windows, because 7z is not using the TZ env var there. timezone = os.environ.update({'TZ': 'GMT'}) args = [ listing, tech_info, arch_type, output_as_utf, password, abs_location ] rc, stdout, _stderr = command.execute(cmd='7z', args=args, env=timezone, root_dir=root_dir, to_files=True) if rc != 0: _error = get_7z_errors(stdout) or 'No error returned' # still try to get the listing? # print(Exception(error)) pass # the listing was produced as UTF on windows to avoid damaging binary # paths in console outputs utf = bool(output_as_utf) return parse_7z_listing(stdout, utf)
def list_entries(location, arch_type='*'): """ List entries from a 7zip-supported archive file at location. Yield Entry tuples. Use the -t* 7z cli type option or the provided arch_type 7z type (can be None). """ assert location abs_location = os.path.abspath(os.path.expanduser(location)) # 7z arguments listing = 'l' # NB: we use t* to ensure that all archive types are honored if not arch_type: arch_type = '' else: arch_type = '-t' + arch_type # pass an empty password so that extraction with passwords WILL fail password = '******' tech_info = '-slt' output_as_utf = '' if on_windows: output_as_utf = '-sccUTF-8' # NB: we force running in the GMT timezone, because 7z is unable to set # the TZ correctly when the archive does not contain TZ info. This does # not work on Windows, because 7z is not using the TZ env var there. timezone = os.environ.update({'TZ': 'GMT'}) args = [listing, tech_info, arch_type, output_as_utf, password, abs_location] rc, stdout, _stderr = command.execute(cmd='7z', args=args, env=timezone, root_dir=root_dir, to_files=True) if rc != 0: _error = get_7z_errors(stdout) or 'No error returned' # still try to get the listing? # print(Exception(error)) pass # the listing was produced as UTF on windows to avoid damaging binary # paths in console outputs utf = bool(output_as_utf) return parse_7z_listing(stdout, utf)
def test_can_process_scan_from_json_scan(self): import scancode from commoncode.command import execute test_dir = self.get_test_loc('livescan/scan') json_file = self.get_temp_file('json') scan_cmd = os.path.join(scancode.root_dir, 'scancode') rc, _stdout, _stderr = execute(scan_cmd, [ '-clip', '--email', '--url', '--strip-root', '--format', 'json', test_dir, json_file ]) assert rc == 0 result_file = self.get_temp_file('.csv') with open(result_file, 'wb') as rf: json2csv.json_scan_to_csv(json_file, rf) expected_file = self.get_test_loc('livescan/expected.csv') check_csvs(result_file, expected_file)
def installed_images(image_id=None): """ Return a list of locally installed Docker images as a three tuple of (name, tag, image id). """ # REPOSITORY TAG IMAGE ID CREATED VIRTUAL SIZE # mine/my-cloud 1.4.0.45 9b63d7aa7390 10 weeks ago 296.9 MB rc, stdout, _stderr = command.execute('docker', ['images', '--no-trunc']) # we just skip if things do not run OK. if rc >= 0: # skip the first header line for line in stdout.splitlines(False)[1:]: # split on spaces and keep the first three elements name, tag, imid = line.split()[:3] imid = get_real_id(imid) logger.info('installed_images: %(name)s, %(tag)s, %(imid)s' % locals()) if not image_id or (image_id and imid.startswith(image_id)): yield name, tag, imid
def _parseinfo(self): """ Parse dwarfdump info section of an elf file. """ dwarfdump_command = 'dwarfdump2' rc, out, err = command.execute(cmd=dwarfdump_command, args=['-i', self.elf_location], root_dir=bin_dir, to_files=True) if rc != 0: raise Exception(open(err).read()) # loop through each returned line passing control to a handler with open(out, 'rb') as lines: for line in lines: line = line.strip() if DCOMP_UNIT_START_RE().match(line): dwarfinfo = DwarfInfo() dwarfinfo.parse(self, lines)
def installed_image_history(image_id): """ Return a list of ordered installed Docker layers ID given an image id as a tuple of (layer id, comment). The history is from latest to oldest. """ # lines format # IMAGE CREATED CREATED BY SIZE COMMENT # 9b63d75a7390 10 weeks ago /bin/sh -c #(nop) CMD [/bin/sh -c /startConfd 0 B rc, stdout, _stderr = command.execute('docker', ['history', '--no-trunc', image_id]) # we just skip if things do not run OK. if rc >= 0: # skip the first header line for line in stdout.splitlines(False)[1:]: # progressively partition on two spaces layer_id, _, right = line.partition(' ') _, _, right = right.strip().partition(' ') layer_command, _, right = right.strip().partition(' ') _size, _, right = right.strip().partition(' ') _comment = right.strip() yield get_real_id(layer_id), layer_command
def get_msi_info(location): """ Run the command `msiinfo suminfo` on the file at `location` and return the results in a dictionary This function requires `msiinfo` to be installed on the system, either by installing the `packagedcode-msiinfo` plugin or by installing `msitools` through a package manager. """ rc, stdout, stderr = execute( cmd_loc=get_msiinfo_bin_location(), args=[ 'suminfo', location, ], ) if stderr: error_message = f'Error encountered when reading MSI information from {location}: ' error_message = error_message + stderr raise MsiinfoException(error_message) return parse_msiinfo_suminfo_output(stdout)
def _collect_and_parse_tags(self): ctags_args = ['--fields=K', '--c-kinds=fp', '-f', '-', self.sourcefile] ctags_temp_dir = fileutils.get_temp_dir(base_dir='ctags') envt = {'TMPDIR': ctags_temp_dir} try: rc, stdo, err = command.execute('ctags', ctags_args, env=envt, root_dir=bin_dir, to_files=True) if rc != 0: raise Exception(open(err).read()) with open(stdo, 'rb') as lines: for line in lines: if 'cannot open temporary file' in line: raise Exception('ctags: cannot open temporary file ' ': Permission denied') if line.startswith('!'): continue line = line.strip() if not line: continue splitted = line.split('\t') if (line.endswith('function\tfile:') or line.endswith('prototype\tfile:')): self.local_functions.append(splitted[0]) elif (line.endswith('function') or line.endswith('prototype')): self.global_functions.append(splitted[0]) finally: fileutils.delete(ctags_temp_dir)
def extract(location, target_dir, arch_type='*'): """ Extract all files from a 7zip-supported archive file at location in the target_dir directory. Return a list of warning messages. Raise exception on errors. `arch_type` is the type of 7zip archive passed to the -t 7zip option. Can be None. """ assert location assert target_dir abs_location = os.path.abspath(os.path.expanduser(location)) abs_target_dir = os.path.abspath(os.path.expanduser(target_dir)) # note: there are some issues with the extraction of debian .deb ar files # see sevenzip bug http://sourceforge.net/p/sevenzip/bugs/1472/ # 7z arguments extract = 'x' yes_to_all = '-y' # NB: we use t* to ensure that all archive types are honored if not arch_type: arch_type = '' else: arch_type = '-t' + arch_type # pass an empty password so that extraction with passwords WILL fail password = '******' # renaming may not behave the same way on all OSes in particular Mac and Windows auto_rename_dupe_names = '-aou' # These things do not work well with p7zip for now: # - ensure that we treat the FS as case insensitive even if it is # this ensure we have consistent names across OSes # case_insensitive = '-ssc-' # - force any console output to be UTF-8 encoded # TODO: add this may be for a UTF output on Windows only # output_as_utf = '-sccUTF-8' # working_tmp_dir = '-w<path>' # NB: we force running in the GMT timezone, because 7z is unable to set # the TZ correctly when the archive does not contain TZ info. This does # not work on Windows, because 7z is not using the TZ env var there. timezone = os.environ.update({'TZ': 'GMT'}) # Note: 7z does extract in the current directory so we cwd to the target dir first args = [extract, yes_to_all, auto_rename_dupe_names, arch_type, password, abs_location] rc, stdout, _stderr = command.execute( cmd='7z', args=args, cwd=abs_target_dir, env=timezone, root_dir=root_dir ) if rc != 0: error = get_7z_errors(stdout) or UNKNOWN_ERROR raise ExtractErrorFailedToExtract(error) extractcode.remove_backslashes_and_dotdots(abs_target_dir) return get_7z_warnings(stdout)