def run(self, artifacts: List[Artifact]) -> List[Artifact]:
logger = logging.getLogger(__name__)
if len(artifacts) == 1:
artifact = artifacts[0]
else:
msg = ('C Compiler expects only one Artifact, '
f'but was given {len(artifacts)}')
raise TaskException(msg)
command = [self._compiler]
command.extend(self._flags)
command.append(str(artifact.location))
output_file = (self._workspace /
artifact.location.with_suffix('.o').name)
command.extend(['-o', str(output_file)])
logger.debug('Running command: ' + ' '.join(command))
subprocess.run(command, check=True)
object_artifact = Artifact(output_file,
BinaryObject,
Compiled)
for definition in artifact.defines:
object_artifact.add_definition(definition)
return [object_artifact]
def run(self, artifacts: List[Artifact]) -> List[Artifact]:
logger = logging.getLogger(__name__)
if len(artifacts) == 1:
artifact = artifacts[0]
else:
msg = ('C Pragma Injector expects only one Artifact, '
f'but was given {len(artifacts)}')
raise TaskException(msg)
logger.debug('Injecting pragmas into: %s', artifact.location)
injector = _CTextReaderPragmas(
FileTextReader(artifact.location))
output_file = self._workspace / artifact.location.name
out_lines = [line for line in injector.line_by_line()]
with output_file.open('w') as out_file:
for line in out_lines:
out_file.write(line)
new_artifact = Artifact(output_file,
artifact.filetype,
Modified)
for dependency in artifact.depends_on:
new_artifact.add_dependency(dependency)
return [new_artifact]
def run(self, artifacts: List[Artifact]) -> List[Artifact]:
logger = logging.getLogger(__name__)
if len(artifacts) == 1:
artifact = artifacts[0]
else:
msg = ('C Preprocessor expects only one Artifact, '
f'but was given {len(artifacts)}')
raise TaskException(msg)
command = [self._preprocessor]
command.extend(self._flags)
command.append(str(artifact.location))
# Use temporary output name (in case the given tool
# can't operate in-place)
output_file = (self._workspace /
artifact.location.with_suffix('.fabcpp').name)
command.append(str(output_file))
logger.debug('Running command: ' + ' '.join(command))
subprocess.run(command, check=True)
# Overwrite actual output file
final_output = (self._workspace /
artifact.location.name)
command = ["mv", str(output_file), str(final_output)]
logger.debug('Running command: ' + ' '.join(command))
subprocess.run(command, check=True)
return [Artifact(final_output,
artifact.filetype,
Raw)]
def CTextReaderPragmas(fpath):
"""
Reads a C source file but when encountering an #include
preprocessor directive injects a special Fab-specific
#pragma which can be picked up later by the Analyser
after the preprocessing
"""
_include_re: str = r'^\s*#include\s+(\S+)'
_include_pattern: Pattern = re.compile(_include_re)
for line in open(fpath, 'rt', encoding='utf-8'):
include_match: Optional[Match] = _include_pattern.match(line)
if include_match:
# For valid C the first character of the matched
# part of the group will indicate whether this is
# a system library include or a user include
include: str = include_match.group(1)
if include.startswith('<'):
yield '#pragma FAB SysIncludeStart\n'
yield line
yield '#pragma FAB SysIncludeEnd\n'
elif include.startswith(('"', "'")):
yield '#pragma FAB UsrIncludeStart\n'
yield line
yield '#pragma FAB UsrIncludeEnd\n'
else:
msg = 'Found badly formatted #include'
raise TaskException(msg)
else:
yield line
def line_by_line(self) -> Iterator[str]:
for line in self._source.line_by_line():
include_match: Optional[Match] \
= self._include_pattern.match(line)
if include_match:
# For valid C the first character of the matched
# part of the group will indicate whether this is
# a system library include or a user include
include: str = include_match.group(1)
# TODO: Is this sufficient? Or do the pragmas
# need to include identifying info
# e.g. the name of the original include?
if include.startswith('<'):
yield '#pragma FAB SysIncludeStart\n'
yield line
yield '#pragma FAB SysIncludeEnd\n'
elif include.startswith(('"', "'")):
yield '#pragma FAB UsrIncludeStart\n'
yield line
yield '#pragma FAB UsrIncludeEnd\n'
else:
msg = 'Found badly formatted #include'
raise TaskException(msg)
else:
yield line
def _compile_file(self, analysed_file: AnalysedFile):
# todo: should really use input_to_output_fpath() here
output_fpath = analysed_file.fpath.with_suffix('.o')
# already compiled?
if self._config.reuse_artefacts and output_fpath.exists():
log_or_dot(logger, f'CompileC skipping: {analysed_file.fpath}')
else:
with Timer() as timer:
output_fpath.parent.mkdir(parents=True, exist_ok=True)
command = self.exe.split()
command.extend(self.flags.flags_for_path(
path=analysed_file.fpath,
source_root=self._config.source_root,
project_workspace=self._config.project_workspace))
command.append(str(analysed_file.fpath))
command.extend(['-o', str(output_fpath)])
log_or_dot(logger, 'CompileC running command: ' + ' '.join(command))
try:
run_command(command)
except Exception as err:
return TaskException(f"error compiling {analysed_file.fpath}: {err}")
send_metric(self.name, str(analysed_file.fpath), timer.taken)
return CompiledFile(analysed_file, output_fpath)
def run(self, artifacts: List[Artifact]) -> List[Artifact]:
logger = logging.getLogger(__name__)
if len(artifacts) == 1:
artifact = artifacts[0]
else:
msg = ('Header Analyser expects only one Artifact, '
f'but was given {len(artifacts)}')
raise TaskException(msg)
new_artifact = Artifact(artifact.location, artifact.filetype,
HeadersAnalysed)
reader = FileTextReader(artifact.location)
logger.debug('Looking for headers in: %s', reader.filename)
for line in reader.line_by_line():
include_match: Optional[Match] \
= self._include_pattern.match(line)
if include_match:
include: str = include_match.group(1)
logger.debug('Found header: %s', include)
if include.startswith(('"', "'")):
include = include.strip('"').strip("'")
logger.debug(' * User header; adding dependency')
new_artifact.add_dependency(Path(self._workspace /
include))
return [new_artifact]
def run(self, artifacts: List[Artifact]) -> List[Artifact]:
if len(artifacts) == 1:
artifact = artifacts[0]
else:
msg = ('C Analyser expects only one Artifact, '
f'but was given {len(artifacts)}')
raise TaskException(msg)
reader = FileTextReader(artifact.location)
state = CWorkingState(self.database)
state.remove_c_file(reader.filename)
new_artifact = Artifact(artifact.location, artifact.filetype, Analysed)
state = CWorkingState(self.database)
state.remove_c_file(reader.filename)
index = clang.cindex.Index.create()
translation_unit = index.parse(reader.filename, args=["-xc"])
# Create include region line mappings
self._locate_include_regions(translation_unit)
# Now walk the actual nodes and find all relevant external symbols
usr_includes = []
current_def = None
for node in translation_unit.cursor.walk_preorder():
if node.kind == clang.cindex.CursorKind.FUNCTION_DECL:
if (node.is_definition()
and node.linkage == clang.cindex.LinkageKind.EXTERNAL):
# This should catch function definitions which are exposed
# to the rest of the application
current_def = CSymbolID(node.spelling, artifact.location)
state.add_c_symbol(current_def)
new_artifact.add_definition(node.spelling)
else:
# Any other declarations should be coming in via headers,
# we can use the injected pragmas to work out whether these
# are coming from system headers or user headers
if (self._check_for_include(
node.location.line) == "usr_include"):
usr_includes.append(node.spelling)
elif (node.kind == clang.cindex.CursorKind.CALL_EXPR):
# When encountering a function call we should be able to
# cross-reference it with a definition seen earlier; and
# if it came from a user supplied header then we will
# consider it a dependency within the project
if node.spelling in usr_includes and current_def is not None:
# TODO: Assumption that the most recent exposed
# definition encountered above is the one which
# should lodge this dependency - is that true?
state.add_c_dependency(current_def, node.spelling)
new_artifact.add_dependency(node.spelling)
return [new_artifact]
def _process_use_statement(self, analysed_file, obj):
use_name = _typed_child(obj, Name)
if not use_name:
raise TaskException("ERROR finding name in use statement:", obj.string)
use_name = use_name.string
if use_name in self.ignore_mod_deps:
logger.debug(f"ignoring use of {use_name}")
elif use_name not in self._intrinsic_modules:
# found a dependency on fortran
analysed_file.add_module_dep(use_name)
def run(self, artifacts: List[Artifact]) -> List[Artifact]:
if len(artifacts) == 1:
artifact = artifacts[0]
else:
msg = ('Fortran Preprocessor expects only one Artifact, '
f'but was given {len(artifacts)}')
raise TaskException(msg)
command = [self._preprocessor]
command.extend(self._flags)
command.append(str(artifact.location))
output_file = (self._workspace /
artifact.location.with_suffix('.f90').name)
command.append(str(output_file))
subprocess.run(command, check=True)
return [Artifact(output_file, artifact.filetype, Raw)]
def run(self, artifacts: List[Artifact]) -> List[Artifact]:
if len(artifacts) < 1:
msg = ('Linker expects at least one Artifact, '
f'but was given {len(artifacts)}')
raise TaskException(msg)
command = [self._linker]
output_file = self._workspace / self._output_filename
command.extend(['-o', str(output_file)])
for artifact in artifacts:
command.append(str(artifact.location))
command.extend(self._flags)
subprocess.run(command, check=True)
return [Artifact(output_file, Executable, Linked)]
def run(self, artifacts: List[Artifact]) -> List[Artifact]:
if len(artifacts) == 1:
artifact = artifacts[0]
else:
msg = ('Header Analyser expects only one Artifact, '
f'but was given {len(artifacts)}')
raise TaskException(msg)
new_artifact = Artifact(artifact.location, artifact.filetype,
HeadersAnalysed)
reader = FileTextReader(artifact.location)
for line in reader.line_by_line():
include_match: Optional[Match] \
= self._include_pattern.match(line)
if include_match:
include: str = include_match.group(1)
if include.startswith(('"', "'")):
include = include.strip('"').strip("'")
new_artifact.add_dependency(Path(self._workspace /
include))
return [new_artifact]
def run(self, artifacts: List[Artifact]) -> List[Artifact]:
logger = logging.getLogger(__name__)
if len(artifacts) == 1:
artifact = artifacts[0]
else:
msg = ('Fortran Analyser expects only one Artifact, '
f'but was given {len(artifacts)}')
raise TaskException(msg)
reader = FileTextReader(artifact.location)
new_artifact = Artifact(artifact.location, artifact.filetype, Analysed)
state = FortranWorkingState(self.database)
state.remove_fortran_file(reader.filename)
normalised_source = FortranNormaliser(reader)
scope: List[Tuple[str, str]] = []
for line in normalised_source.line_by_line():
logger.debug(scope)
logger.debug('Considering: %s', line)
if len(scope) == 0:
unit_match: Optional[Match] \
= self._program_unit_pattern.match(line)
if unit_match:
unit_type: str = unit_match.group(1).lower()
unit_name: str = unit_match.group(2).lower()
logger.debug('Found %s called "%s"', unit_type, unit_name)
unit_id = FortranUnitID(unit_name, reader.filename)
state.add_fortran_program_unit(unit_id)
new_artifact.add_definition(unit_name)
scope.append((unit_type, unit_name))
continue
use_match: Optional[Match] \
= self._use_pattern.match(line)
if use_match:
use_name: str = use_match.group(3).lower()
if use_name in self._intrinsic_modules:
logger.debug('Ignoring intrinsic module "%s"', use_name)
else:
if len(scope) == 0:
use_message \
= '"use" statement found outside program unit'
raise TaskException(use_message)
logger.debug('Found usage of "%s"', use_name)
unit_id = FortranUnitID(scope[0][1], reader.filename)
state.add_fortran_dependency(unit_id, use_name)
new_artifact.add_dependency(use_name)
continue
block_match: Optional[Match] = self._scoping_pattern.match(line)
if block_match:
# Beware we want the value of a different group to the one we
# check the presence of.
#
block_name: str = block_match.group(1) \
and block_match.group(2).lower()
block_nature: str = block_match.group(3).lower()
logger.debug('Found %s called "%s"', block_nature, block_name)
scope.append((block_nature, block_name))
continue
proc_match: Optional[Match] \
= self._procedure_pattern.match(line)
if proc_match:
proc_nature = proc_match.group(1).lower()
proc_name = proc_match.group(2).lower()
logger.debug('Found %s called "%s"', proc_nature, proc_name)
# Note: We append a tuple so double brackets.
scope.append((proc_nature, proc_name))
continue
iface_match: Optional[Match] = self._interface_pattern.match(line)
if iface_match:
iface_name = iface_match.group(1) \
and iface_match.group(1).lower()
logger.debug('Found interface called "%s"', iface_name)
scope.append(('interface', iface_name))
continue
type_match: Optional[Match] = self._type_pattern.match(line)
if type_match:
type_name = type_match.group(3).lower()
logger.debug('Found type called "%s"', type_name)
scope.append(('type', type_name))
continue
end_match: Optional[Match] = self._end_block_pattern.match(line)
if end_match:
end_nature: str = end_match.group(1) \
and end_match.group(1).lower()
end_name: str = end_match.group(2) \
and end_match.group(2).lower()
logger.debug('Found end of %s called %s', end_nature, end_name)
exp: Tuple[str, str] = scope.pop()
if end_nature is not None:
if end_nature != exp[0]:
end_message = 'Expected end of {exp} "{name}" ' \
'but found {found}'
end_values = {
'exp': exp[0],
'name': exp[1],
'found': end_nature
}
raise TaskException(end_message.format(**end_values))
if end_name is not None:
if end_name != exp[1]:
end_message = 'Expected end of {exp} "{name}" ' \
'but found end of {found}'
end_values = {
'exp': exp[0],
'name': exp[1],
'found': end_name
}
raise TaskException(end_message.format(**end_values))
return [new_artifact]
def run(self, artifacts: List[Artifact]) -> List[Artifact]:
logger = logging.getLogger(__name__)
if len(artifacts) == 1:
artifact = artifacts[0]
else:
msg = ('Fortran Analyser expects only one Artifact, '
f'but was given {len(artifacts)}')
raise TaskException(msg)
reader = FileTextReader(artifact.location)
new_artifact = Artifact(artifact.location, artifact.filetype, Analysed)
state = FortranWorkingState(self.database)
state.remove_fortran_file(reader.filename)
logger.debug('Analysing: %s', reader.filename)
# If this file defines any C symbol bindings it may also
# end up with an entry in the C part of the database
cstate = CWorkingState(self.database)
cstate.remove_c_file(reader.filename)
normalised_source = FortranNormaliser(reader)
scope: List[Tuple[str, str]] = []
for line in normalised_source.line_by_line():
logger.debug(scope)
logger.debug('Considering: %s', line)
if len(scope) == 0:
unit_match: Optional[Match] \
= self._program_unit_pattern.match(line)
if unit_match is not None:
unit_type: str = unit_match.group(1).lower()
unit_name: str = unit_match.group(2).lower()
logger.debug('Found %s called "%s"', unit_type, unit_name)
unit_id = FortranUnitID(unit_name, reader.filename)
state.add_fortran_program_unit(unit_id)
new_artifact.add_definition(unit_name)
scope.append((unit_type, unit_name))
continue
use_match: Optional[Match] \
= self._use_pattern.match(line)
if use_match is not None:
use_name: str = use_match.group(3).lower()
if use_name in self._intrinsic_modules:
logger.debug('Ignoring intrinsic module "%s"', use_name)
else:
if len(scope) == 0:
use_message \
= '"use" statement found outside program unit'
raise TaskException(use_message)
logger.debug('Found usage of "%s"', use_name)
unit_id = FortranUnitID(scope[0][1], reader.filename)
state.add_fortran_dependency(unit_id, use_name)
new_artifact.add_dependency(use_name)
continue
block_match: Optional[Match] = self._scoping_pattern.match(line)
if block_match is not None:
# Beware we want the value of a different group to the one we
# check the presence of.
#
block_name: str = block_match.group(1) \
and block_match.group(2).lower()
block_nature: str = block_match.group(3).lower()
logger.debug('Found %s called "%s"', block_nature, block_name)
scope.append((block_nature, block_name))
continue
proc_match: Optional[Match] \
= self._procedure_pattern.match(line)
if proc_match is not None:
proc_nature = proc_match.group(1).lower()
proc_name = proc_match.group(2).lower()
logger.debug('Found %s called "%s"', proc_nature, proc_name)
scope.append((proc_nature, proc_name))
# Check for the procedure being symbol-bound to C
cbind_match: Optional[Match] \
= self._cbind_pattern.match(line)
if cbind_match is not None:
cbind_name = cbind_match.group(2)
# The name keyword on the bind statement is optional.
# If it doesn't exist, the procedure name is used
if cbind_name is None:
cbind_name = proc_name
cbind_name = cbind_name.lower().strip("'\"")
logger.debug('Bound to C symbol "%s"', cbind_name)
# A bind within an interface block means this is
# exposure of a C-defined function to Fortran,
# otherwise it is going the other way (allowing C
# code to call the Fortran procedure)
if any([stype == "interface" for stype, _ in scope]):
# TODO: This is sort of hijacking the mechanism used
# for Fortran module dependencies, only using the
# symbol name. Longer term we probably need a more
# elegant solution
logger.debug('In an interface block; so a dependency')
unit_id = FortranUnitID(scope[0][1], reader.filename)
state.add_fortran_dependency(unit_id, cbind_name)
new_artifact.add_dependency(cbind_name)
else:
# Add to the C database
logger.debug('Not an interface block; so a definition')
symbol_id = CSymbolID(cbind_name, reader.filename)
cstate.add_c_symbol(symbol_id)
new_artifact.add_definition(cbind_name)
continue
cbind_match = self._cbind_pattern.match(line)
if cbind_match is not None:
# This should be a line binding from C to a variable definition
# (procedure binds are dealt with above)
cbind_name = cbind_match.group(2)
# The name keyword on the bind statement is optional.
# If it doesn't exist, the Fortran variable name is used
if cbind_name is None:
var_search = re.search(r'.*::\s*(\w+)', line)
if var_search:
cbind_name = var_search.group(1)
else:
cbind_message \
= 'failed to find variable name ' \
'on C bound variable'
raise TaskException(cbind_message)
cbind_name = cbind_name.lower().strip("'\"")
logger.debug('Found C bound variable called "%s"', cbind_name)
# Add to the C database
symbol_id = CSymbolID(cbind_name, reader.filename)
cstate.add_c_symbol(symbol_id)
new_artifact.add_definition(cbind_name)
iface_match: Optional[Match] = self._interface_pattern.match(line)
if iface_match is not None:
iface_name = iface_match.group(1) \
and iface_match.group(1).lower()
logger.debug('Found interface called "%s"', iface_name)
scope.append(('interface', iface_name))
continue
type_match: Optional[Match] = self._type_pattern.match(line)
if type_match is not None:
type_name = type_match.group(3).lower()
logger.debug('Found type called "%s"', type_name)
scope.append(('type', type_name))
continue
end_match: Optional[Match] = self._end_block_pattern.match(line)
if end_match is not None:
end_nature: str = end_match.group(1) \
and end_match.group(1).lower()
end_name: str = end_match.group(2) \
and end_match.group(2).lower()
logger.debug('Found end of %s called %s', end_nature, end_name)
exp: Tuple[str, str] = scope.pop()
if end_nature is not None:
if end_nature != exp[0]:
end_message = 'Expected end of {exp} "{name}" ' \
'but found {found}'
end_values = {
'exp': exp[0],
'name': exp[1],
'found': end_nature
}
raise TaskException(end_message.format(**end_values))
if end_name is not None:
if end_name != exp[1]:
end_message = 'Expected end of {exp} "{name}" ' \
'but found end of {found}'
end_values = {
'exp': exp[0],
'name': exp[1],
'found': end_name
}
raise TaskException(end_message.format(**end_values))
return [new_artifact]