def run_etags(root, files, chunk=2000):
"""Run etags from the given root over the given files.
Some operating systems limit the number of command line arguments
(or the length of the command), we we process the list of files in
chunks.
"""
if not files:
return ''
try:
os.remove(TAGS)
except FileNotFoundError:
pass # file did not exist
logging.info('Running etags on %s files...', len(files))
while files:
paths = files[:chunk]
files = files[chunk:]
logging.info('Running etags on %s files starting with %s...',
len(paths), paths[0])
runt.run([ETAGS, '-o', TAGS, '--append'] + paths,
cwd=root,
encoding='latin1')
logging.info('Finished running etags.')
with open(os.path.join(root, TAGS)) as etags_file:
etags_data = etags_file.read()
os.remove(os.path.join(root, TAGS))
return etags_data
def find_sources(root, exclude, extensions):
"""Use find to list the source files under root."""
logging.info('Running find...')
cmd = ['find', '-L', '.']
files = runt.run(cmd, root).strip().splitlines()
logging.info('Running find...done')
return select_source_files(files, root, exclude, extensions)
def have_etags():
"""Test for the existence of etags."""
try:
help_banner = runt.run([ETAGS,
'--help']).splitlines()[0].lower().split()
return ETAGS in help_banner
except FileNotFoundError:
return False
def have_ctags():
"""Test for existence of exuberant ctags."""
try:
help_banner = runt.run([CTAGS,
'--help']).splitlines()[0].lower().split()
return all(string in help_banner for string in [EXUBERANT, CTAGS])
except FileNotFoundError:
return False
def __init__(self, goto, root, cwd=None):
cmd = ['cbmc', '--show-loops', '--json-ui', goto]
try:
super(LoopFromGoto, self).__init__(
[parse_cbmc_json(json.loads(runt.run(cmd, cwd=cwd)), root)])
except subprocess.CalledProcessError as err:
raise UserWarning('Failed to run {}: {}'.format(cmd, str(err)))
except json.decoder.JSONDecodeError as err:
raise UserWarning('Failed to parse output of {}: {}'.format(
cmd, str(err)))
def find_sources(self, build):
"""Use make to list the source files used to build a goto binary."""
# Remove object files
runt.run(['make', 'clean'], build)
# Build with the preprocessor
preprocessor_commands = self.build_with_preprocessor(build)
logging.debug('preprocessor commands: %s', preprocessor_commands)
preprocessed_filenames = self.extract_filenames(
preprocessor_commands, build)
logging.debug('preprocessed filenames: %s', preprocessed_filenames)
preprocessed_output = self.read_output(preprocessed_filenames)
#logging.debug('preprocessed output: %s', preprocessed_output)
source_files = self.extract_source_filenames(preprocessed_output,
build)
logging.debug('source files: %s', source_files)
# Remove preprocessor output
runt.run(['make', 'clean'], build)
return source_files
def __init__(self, goto, root, cwd=None):
cmd = ["goto-analyzer", "--reachable-functions", "--json", "-", goto]
try:
analyzer_output = runt.run(cmd, cwd=cwd)
except subprocess.CalledProcessError as err:
raise UserWarning('Failed to run {}: {}'.format(cmd,
str(err))) from err
json_data = parse.parse_json_string(analyzer_output,
fail=True,
goto_analyzer=True)
super().__init__([parse_cbmc_json(json_data, root)])
def symbol_table(goto):
"""Extract symbol table from goto binary as lines of text."""
# The --show-symbol-table flag produces a sequence of symbol
# definitions. Definitions are separated by blank lines. Each
# definition is a sequence of lines including
#
# Symbol......: symbol_name
# Pretty name.: simple_symbol_name
# Location....: file file_name line line_number
cmd = ['cbmc', '--show-symbol-table', goto]
definitions = re.split(r'[\n\r][\n\r]+', runt.run(cmd))
return [definition.strip().splitlines() for definition in definitions]
def run_ctags(root, files, chunk=2000):
"""Run ctags from the given root over the given files.
Some operating systems limit the number of command line arguments
(or the length of the command), we we process the list of files in
chunks.
"""
if not files:
return ''
logging.info('Running ctags on %s files...', len(files))
output = ''
while files:
paths = files[:chunk]
files = files[chunk:]
logging.info('Running ctags on %s files starting with %s...',
len(paths), paths[0])
output += runt.run([CTAGS, '-x'] + paths, cwd=root, encoding='latin1')
logging.info('Running ctags on %s files...done', len(files))
return output
def symbol_table(goto):
"""Extract symbol table from goto binary as lines of text."""
# The --show-symbol-table flag produces a sequence of symbol
# definitions. Definitions are separated by blank lines. Each
# definition is a sequence of lines including
#
# Symbol......: symbol_name
# Pretty name.: simple_symbol_name
# Location....: file file_name line line_number
# Code with definitions like
# static const char *UTF_16_BE_BOM = "\xFE\xFF";
# will produce values in the symbol table that cannot be read as
# strings of UTF-8 characters. We read the symbol table using the
# latin1 encoding in place of the Python default UTF-8 encoding,
# since latin1 agrees with UTF-8 on the ASCII characters.
cmd = ['cbmc', '--show-symbol-table', goto]
definitions = re.split(r'[\n\r][\n\r]+', runt.run(cmd, encoding='latin1'))
return [definition.strip().splitlines() for definition in definitions]
def build_with_preprocessor(build):
"""Build the goto binary using goto-cc as a preprocessor.
Return the list of goto-cc commands used in the build.
"""
# Make will fail when it tries to link the preprocessed output
# What is a system-independent way of skipping the link failure?
# For now, we assume error code 2 is generated by the link failure.
# build the project with the preprocessor and capture the make output
result = runt.run(['make', 'GOTO_CC=goto-cc -E', 'goto'],
build,
ignored=[2])
# strip line continuations in the make output
result = result.replace('\\\n', '')
# return the invocations of goto-cc in the make output
return [
line.strip() for line in result.splitlines()
if line.strip().startswith('goto-cc')
]
def sloc(files, root):
"""Run sloc on a list of files under root."""
loc = {}
sources = files
while sources:
files = sources[:100]
sources = sources[100:]
logging.info('Running sloc on %s files starting with %s...',
len(files), files[0])
command = ['sloc', '-f', 'json'] + files
logging.debug('sloc: %s', ' '.join(command))
try:
result = runt.run(command, root)
except FileNotFoundError as error:
# handle sloc command not found
logging.info('sloc not found: %s', error.strerror)
return None
except OSError as error:
# handle sloc argument list too long
if error.errno != 7: # errno==7 is 'Argument list too long'
raise
logging.info('sloc argument list too long: %s', error.strerror)
return None
except subprocess.CalledProcessError as error:
# handle sloc error generated by running sloc
logging.info('Unable to run sloc: %s', error.stderr)
return None
# sloc produces useful data in addition to the summary
data = json.loads(result)['summary']
for key, val in data.items():
loc[key] = int(val) + loc.get(key, 0)
return loc