def _extract_from_unit(self, unit):
if unit.root is None:
with log_stdout('error'):
print('Could not parse {}:'.format(unit.filename))
for diag in unit.diagnostics:
print(' {}'.format(diag))
return
unit.populate_lexical_env()
subpdata = analysis.traverse_unit(unit.root)
self.subpdata.update(subpdata)
progs = []
for subp, subpuserdata in subpdata.iteritems():
if subp.is_a(lal.BaseSubpBody):
start_t = time.clock()
progs.append(
gen_ir(self, subp, self._internal_typer, subpuserdata)
)
end_t = time.clock()
log(
'timings',
" - Transformation of subprocedure {} took {}s".format(
subp.f_subp_spec.f_subp_name.text,
end_t - start_t
)
)
converter = ConvertUniversalTypes(self.evaluator, self._internal_typer)
for prog in progs:
prog.visit(converter)
return progs
def create_provider_config(args, analysis_files):
"""
Creates a ProviderConfig object using the switches have been passed
as argument, among "-P", "-X", "--target", "--provider-files[-from]".
Note that if the "--provider-files[-from]" switches have not been
specified, it uses the set of files to analyze as provider files.
:param argparse.Namespace args: The command-line arguments.
:param list[str] analysis_files: The files to analyze.
:rtype: ProviderConfig
"""
project_file = args.P
scenario_vars = dict([eq.split('=') for eq in args.X])
provider_files = set(
commands_from_file_or_list(args.provider_files_from,
args.provider_files) or [])
target = args.target
# Make sure that files to analyze are part of the auto provider.
provider_files.update(analysis_files)
if project_file is None and len(scenario_vars) > 0:
logger.log('error', "warning: use of scenario vars without a "
"project file.")
if project_file is None and target is not None:
logger.log('error', "warning: specifying a target without a "
"project file.")
return ProviderConfig(project_file=project_file,
scenario_vars=tuple(scenario_vars.iteritems()),
provider_files=tuple(provider_files),
target=target)
def on_subgoal_achieved(subgoal):
"""
Function that will be called as soon as a subgoal is achieved in during
the execution of the schedule.
:type subgoal: object
"""
if isinstance(subgoal, AnalysisOfFile):
logger.log('progress', 'analyzed {}'.format(subgoal.filename))
def run(self, ctx):
try:
unit = ctx.get_from_file(self.filename)
if unit.root is not None:
return {'res': unit}
log('error', '\n'.join(str(diag) for diag in unit.diagnostics))
except Exception as e:
with log_stdout('info'):
print('error: libadalang failed to analyze {}: {}.'.format(
self.filename, e))
traceback.print_exc(file=sys.stdout)
return {'res': None}
def on_timeout(file_cause):
"""
Function that will be called if one of the partition times out.
:param string file_cause: The filename of the file that caused the time
out. We know that the cause will be a file in this case because that
is how checkers are using the timeout mechanism.
"""
if file_cause is not None:
error_msg = "process timed out due to file {}.".format(file_cause)
else:
error_msg = "process timed out."
logger.log('error', error_msg)
def lines_from_file(filename):
"""
Returns the list of lines of the file.
If the file cannot be opened, an error is logged and an empty list is
returned.
:param str filename: The path to the file to read.
:rtype: list[str]
"""
try:
with open(filename) as f:
return [l.strip() for l in f.readlines()]
except EnvironmentError:
logger.log('error', 'error: cannot read file {}'.format(filename))
return []
def run(self, **sem):
for i, f in enumerate(self.filenames):
if self._c_file_matcher.match(f):
sem_f = sem[str(i)]
for sem_subp in sem_f:
subp = sem_subp.orig_subp
subp_name = subp.f_subp_spec.f_subp_name.text
if self._c_subp_matcher.match(subp_name):
out_file = self._actual_filename(f, subp_name)
self._ensure_dir(out_file)
sem_subp.save_results_to_file(out_file)
log('info',
'Saved analysis results to {}'.format(out_file))
return {'res': []}
def run(self, ctx, unit):
irtree = None
if unit is not None:
log('info', 'Transforming {}'.format(self.filename))
try:
start_t = time.clock()
irtree = ctx.extract_programs_from_unit(unit)
end_t = time.clock()
log(
'timings', "Transformation of {} took {}s.".format(
self.filename, end_t - start_t))
except Exception as e:
with log_stdout('info'):
print(
'error: could not generate IR for file {}: {}.'.format(
self.filename, e))
traceback.print_exc(file=sys.stdout)
return {'res': irtree}
def do_partition(args, provider_config, checkers, partition):
"""
Runs the checkers on a single partition of the whole set of files.
Returns a list of diagnostics.
:param argparse.Namespace args: The command-line arguments.
:param ProviderConfig provider_config: The provider configuration.
:param list[(Checker, list[str])] checkers: The list of checkers to run
together with their specific arguments.
:param (int, list[str]) partition: The index of that partition and the list
of files that make up that partition.
:rtype: list[(DiagnosticPosition, str, MessageKind, str)]
"""
set_logger(args)
keepalive(2)
diags = []
index, files = partition
logger.log('debug',
"started partition {} with files {}".format(index, files))
try:
reqs = get_requirements(provider_config, checkers, files)
schedule = get_schedules(reqs)[0]
if args.export_schedule is not None:
export_schedule(schedule, "{}{}".format(args.export_schedule,
index))
for checker_result in schedule.run(on_subgoal_achieved).values():
for program_result in checker_result:
for diag in program_result.diagnostics:
report = program_result.diag_report(diag)
if report is not None:
diags.append(report)
except Exception:
with logger.log_stdout('internal-error'):
traceback.print_exc(file=sys.stdout)
finally:
logger.log('debug', "completed partition {}".format(index))
return diags
def _ensure_dir(filename):
"""
Ensures that the directories along the file path exist, creating them
along the way when they don't.
:param str filename: path of the file.
"""
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
os.makedirs(dirname)
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
log(
'error',
'Could not create directories along {}'.format(
dirname))
raise
elif os.path.isfile(dirname):
log('error',
'Path {} is already the path of a file'.format(dirname))
def run(self, ir, model_and_merge_pred):
res = []
if ir is not None and model_and_merge_pred is not None:
log('info', 'Analyzing file {}'.format(self.analysis_file))
model, merge_pred_builder = model_and_merge_pred
start_t = time.clock()
for prog in ir:
fun = prog.data.fun_id
subp_start_t = time.clock()
try:
res.append(
abstract_analysis.compute_semantics(
prog, model[prog], merge_pred_builder))
except Exception as e:
with log_stdout('info'):
print('error: analysis of subprocedure {}({}) failed: '
'{}.'.format(fun.f_subp_spec.f_subp_name.text,
fun.sloc_range, e))
traceback.print_exc(file=sys.stdout)
subp_end_t = time.clock()
log(
'timings',
" - Analysis of subprocedure {} took {}s.".format(
fun.f_subp_spec.f_subp_name.text,
subp_end_t - subp_start_t))
end_t = time.clock()
log(
'timings',
"Analysis of {} took {}s.".format(self.analysis_file,
end_t - start_t))
log('progress', 'analyzed {}'.format(self.analysis_file))
return {'res': res}
def get_working_checkers(args):
"""
Retrieves the list of checkers to run from the information passed as
command-line arguments using the --checkers or --checkers-from options.
The return value is a list of pairs of (Checker, list[str]) corresponding
to the actual checker objects to run together with a list of arguments
specific to each checker run. Additionnally, a boolean is returned to
indicate whether all checkers were successfully loaded or not.
Errors may be reported if the specified checkers are not found or do
not export the checker interface.
:param argparse.Namespace args: The command-line arguments.
:rtype: list[(Checker, list[str])], bool
"""
checker_commands = commands_from_file_or_list(args.checkers_from,
args.checkers)
split_commands = [command.split() for command in checker_commands]
checkers = []
for checker_args in split_commands:
# checker_args[0] is the python module to the checker. The rest are
# additional arguments that must be passed to that checker.
try:
checker_module = import_checker(checker_args[0])
except ImportError:
logger.log(
'error',
'Failed to import checker module {}.'.format(checker_args[0]))
else:
if not hasattr(checker_module, 'checker'):
logger.log(
'error', 'Checker {} does not export a "checker" '
'object.'.format(checker_module))
elif not issubclass(checker_module.checker, Checker):
logger.log(
'error', 'Checker {} does not inherit the '
'"lalcheck.checkers.support.checker.Checker" '
'interface.'.format(checker_module))
else:
checkers.append((checker_module.checker, checker_args[1:]))
return checkers, len(checkers) == len(checker_commands)
def do_all(args, diagnostic_action):
"""
Main routine of the driver. Uses the user-provided arguments to run
checkers on a list of files. Takes care of parallelizing the process,
distributing the tasks, etc.
:param argparse.Namespace args: The command-line arguments.
:param str diagnostic_action: Action to perform with the diagnostics that
are produced by the checkers. Can be one of:
- 'return': Return them as a list.
- 'log': Output them in the logger.
"""
args.j = cpu_count() if args.j <= 0 else args.j
working_files = sort_files_by_line_count(
commands_from_file_or_list(args.files_from, args.files))
provider_config = create_provider_config(args, working_files)
ps = args.partition_size
ps = len(working_files) / args.j if ps == 0 else ps
ps = max(ps, 1)
def compute_partitions():
# Input: list of files sorted by line count: [file_1, ..., file_n]
# Step 1: Distribute them among the j cores: [
# [file_1, file_{1+j}, file_{1+2j}, ...],
# [file_2, file_{2+j}, file_{2+2j}, ...],
# ...,
# [file_j, file_{2j}, file_{3j}, ...]
# ]
process_parts = [working_files[i::args.j] for i in range(args.j)]
# Step 2: Split each in lists of maximum length ps: [
# [[file_1, ..., file_{1+(ps-1)*j}], [file_{1+ps*j}, ...], ...}]],
# [[file_2, ..., file_{2+(ps-1)*j}], [file_{2+ps*j}, ...], ...],
# ...,
# [[file_j, ..., file_{ps*j}], [file_{(ps+1)*j}, ...], ...]
# ]
split_parts = [[part[i:i + ps] for i in range(0, len(part), ps)]
for part in process_parts]
# Final step: Transpose and flatten the above matrix to make sure each
# core gets a partition of size ps: [
# [file_1, ..., file_{1+(ps-1)*j}],
# [file_2, ..., file_{2+(ps-1)*j}],
# ...,
# [file_j, ..., file_{ps*j}],
#
# ...,
#
# [file_{1+ps*j}, ...],
# [file_{2+ps*j}, ...],
# ...,
# [file_{(ps+1)*j}, ...],
#
# ...
# ]
return [x for p in izip_longest(*split_parts, fillvalue=[]) for x in p]
partitions = compute_partitions()
checkers, checker_loading_success = get_working_checkers(args)
if not checker_loading_success:
logger.log('error', 'Some checkers could not be loaded, exiting.')
sys.exit(1)
if args.list_categories:
list_categories(checkers)
if args.checkers_help:
print_checkers_help(checkers)
logger.log(
'info',
'Created {} partitions of {} files.'.format(len(partitions), ps))
logger.log(
'info',
'Parallel analysis done in batches of {} partitions'.format(args.j))
# To handle Keyboard interrupt, the child processes must inherit the
# SIG_IGN (ignore signal) handler from the parent process. (see
# https://stackoverflow.com/a/35134329)
orig_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
p = Pool(args.j, maxtasksperchild=1)
# Restore the original handler of the parent process.
signal.signal(signal.SIGINT, orig_handler)
all_diags = p.map_async(partial(do_partition, args, provider_config,
checkers),
enumerate(partitions),
chunksize=1)
# Keyboard interrupts are ignored if we use wait() or get() without any
# timeout argument. By using this loop, we mimic the behavior of an
# infinite timeout but allow keyboard interrupts to go through.
while not all_diags.ready():
all_diags.wait(1)
p.close()
p.join()
reports = []
for diags in all_diags.get():
for diag in diags:
if diagnostic_action == 'log':
logger.log('diag-{}'.format(diag[3]), report_diag(args, diag))
elif diagnostic_action == 'return':
reports.append(diag)
return reports
def do_all(args, diagnostic_action):
"""
Main routine of the driver. Uses the user-provided arguments to run
checkers on a list of files. Takes care of parallelizing the process,
distributing the tasks, etc.
:param argparse.Namespace args: The command-line arguments.
:param str diagnostic_action: Action to perform with the diagnostics that
are produced by the checkers. Can be one of:
- 'return': Return them as a list.
- 'log': Output them in the logger.
"""
args.j = cpu_count() if args.j <= 0 else args.j
working_files = sort_files_by_line_count(
commands_from_file_or_list(args.files_from, args.files))
provider_config = create_provider_config(args, working_files)
ps = args.partition_size
ps = len(working_files) / args.j if ps == 0 else ps
ps = max(ps, 1)
def compute_partitions():
# Input: list of files sorted by line count: [file_1, ..., file_n]
# Step 1: Distribute them among the j cores: [
# [file_1, file_{1+j}, file_{1+2j}, ...],
# [file_2, file_{2+j}, file_{2+2j}, ...],
# ...,
# [file_j, file_{2j}, file_{3j}, ...]
# ]
process_parts = [working_files[i::args.j] for i in range(args.j)]
# Step 2: Split each in lists of maximum length ps: [
# [[file_1, ..., file_{1+(ps-1)*j}], [file_{1+ps*j}, ...], ...}]],
# [[file_2, ..., file_{2+(ps-1)*j}], [file_{2+ps*j}, ...], ...],
# ...,
# [[file_j, ..., file_{ps*j}], [file_{(ps+1)*j}, ...], ...]
# ]
split_parts = [[part[i:i + ps] for i in range(0, len(part), ps)]
for part in process_parts]
# Final step: Transpose and flatten the above matrix to make sure each
# core gets a partition of size ps: [
# [file_1, ..., file_{1+(ps-1)*j}],
# [file_2, ..., file_{2+(ps-1)*j}],
# ...,
# [file_j, ..., file_{ps*j}],
#
# ...,
#
# [file_{1+ps*j}, ...],
# [file_{2+ps*j}, ...],
# ...,
# [file_{(ps+1)*j}, ...],
#
# ...
# ]
return [x for p in izip_longest(*split_parts, fillvalue=[]) for x in p]
partitions = compute_partitions()
checkers, checker_loading_success = get_working_checkers(args)
if not checker_loading_success:
logger.log('error', 'Some checkers could not be loaded, exiting.')
sys.exit(1)
if args.list_categories or args.list_categories_with_debts:
list_categories(args, checkers, args.list_categories_with_debts)
if args.checkers_help:
print_checkers_help(checkers)
logger.log(
'info',
'Created {} partitions of {} files.'.format(len(partitions), ps))
logger.log(
'info',
'Parallel analysis done in batches of {} partitions'.format(args.j))
if args.no_multiprocessing:
all_diags = []
for element in enumerate(partitions):
all_diags.append(
do_partition(args, provider_config, checkers, element))
else:
all_diags = parallel_map(process_count=args.j,
target=partial(do_partition, args,
provider_config, checkers),
elements=enumerate(partitions),
timeout_factor=args.timeout_factor,
timeout_callback=on_timeout,
readjust=readjust_partition)
reports = []
for diags in all_diags:
for diag in diags:
if diagnostic_action == 'log':
logger.log('diag-{}'.format(diag[3]), report_diag(args, diag))
elif diagnostic_action == 'return':
reports.append(diag)
return reports
