def run(self, artifacts: List[Artifact]) -> List[Artifact]: logger = logging.getLogger(__name__) if len(artifacts) == 1: artifact = artifacts[0] else: msg = ('C Compiler expects only one Artifact, ' f'but was given {len(artifacts)}') raise TaskException(msg) command = [self._compiler] command.extend(self._flags) command.append(str(artifact.location)) output_file = (self._workspace / artifact.location.with_suffix('.o').name) command.extend(['-o', str(output_file)]) logger.debug('Running command: ' + ' '.join(command)) subprocess.run(command, check=True) object_artifact = Artifact(output_file, BinaryObject, Compiled) for definition in artifact.defines: object_artifact.add_definition(definition) return [object_artifact]
def run(self, artifacts: List[Artifact]) -> List[Artifact]: logger = logging.getLogger(__name__) if len(artifacts) == 1: artifact = artifacts[0] else: msg = ('C Pragma Injector expects only one Artifact, ' f'but was given {len(artifacts)}') raise TaskException(msg) logger.debug('Injecting pragmas into: %s', artifact.location) injector = _CTextReaderPragmas( FileTextReader(artifact.location)) output_file = self._workspace / artifact.location.name out_lines = [line for line in injector.line_by_line()] with output_file.open('w') as out_file: for line in out_lines: out_file.write(line) new_artifact = Artifact(output_file, artifact.filetype, Modified) for dependency in artifact.depends_on: new_artifact.add_dependency(dependency) return [new_artifact]
def run(self, artifacts: List[Artifact]) -> List[Artifact]: logger = logging.getLogger(__name__) if len(artifacts) == 1: artifact = artifacts[0] else: msg = ('C Preprocessor expects only one Artifact, ' f'but was given {len(artifacts)}') raise TaskException(msg) command = [self._preprocessor] command.extend(self._flags) command.append(str(artifact.location)) # Use temporary output name (in case the given tool # can't operate in-place) output_file = (self._workspace / artifact.location.with_suffix('.fabcpp').name) command.append(str(output_file)) logger.debug('Running command: ' + ' '.join(command)) subprocess.run(command, check=True) # Overwrite actual output file final_output = (self._workspace / artifact.location.name) command = ["mv", str(output_file), str(final_output)] logger.debug('Running command: ' + ' '.join(command)) subprocess.run(command, check=True) return [Artifact(final_output, artifact.filetype, Raw)]
def CTextReaderPragmas(fpath): """ Reads a C source file but when encountering an #include preprocessor directive injects a special Fab-specific #pragma which can be picked up later by the Analyser after the preprocessing """ _include_re: str = r'^\s*#include\s+(\S+)' _include_pattern: Pattern = re.compile(_include_re) for line in open(fpath, 'rt', encoding='utf-8'): include_match: Optional[Match] = _include_pattern.match(line) if include_match: # For valid C the first character of the matched # part of the group will indicate whether this is # a system library include or a user include include: str = include_match.group(1) if include.startswith('<'): yield '#pragma FAB SysIncludeStart\n' yield line yield '#pragma FAB SysIncludeEnd\n' elif include.startswith(('"', "'")): yield '#pragma FAB UsrIncludeStart\n' yield line yield '#pragma FAB UsrIncludeEnd\n' else: msg = 'Found badly formatted #include' raise TaskException(msg) else: yield line
def line_by_line(self) -> Iterator[str]: for line in self._source.line_by_line(): include_match: Optional[Match] \ = self._include_pattern.match(line) if include_match: # For valid C the first character of the matched # part of the group will indicate whether this is # a system library include or a user include include: str = include_match.group(1) # TODO: Is this sufficient? Or do the pragmas # need to include identifying info # e.g. the name of the original include? if include.startswith('<'): yield '#pragma FAB SysIncludeStart\n' yield line yield '#pragma FAB SysIncludeEnd\n' elif include.startswith(('"', "'")): yield '#pragma FAB UsrIncludeStart\n' yield line yield '#pragma FAB UsrIncludeEnd\n' else: msg = 'Found badly formatted #include' raise TaskException(msg) else: yield line
def _compile_file(self, analysed_file: AnalysedFile): # todo: should really use input_to_output_fpath() here output_fpath = analysed_file.fpath.with_suffix('.o') # already compiled? if self._config.reuse_artefacts and output_fpath.exists(): log_or_dot(logger, f'CompileC skipping: {analysed_file.fpath}') else: with Timer() as timer: output_fpath.parent.mkdir(parents=True, exist_ok=True) command = self.exe.split() command.extend(self.flags.flags_for_path( path=analysed_file.fpath, source_root=self._config.source_root, project_workspace=self._config.project_workspace)) command.append(str(analysed_file.fpath)) command.extend(['-o', str(output_fpath)]) log_or_dot(logger, 'CompileC running command: ' + ' '.join(command)) try: run_command(command) except Exception as err: return TaskException(f"error compiling {analysed_file.fpath}: {err}") send_metric(self.name, str(analysed_file.fpath), timer.taken) return CompiledFile(analysed_file, output_fpath)
def run(self, artifacts: List[Artifact]) -> List[Artifact]: logger = logging.getLogger(__name__) if len(artifacts) == 1: artifact = artifacts[0] else: msg = ('Header Analyser expects only one Artifact, ' f'but was given {len(artifacts)}') raise TaskException(msg) new_artifact = Artifact(artifact.location, artifact.filetype, HeadersAnalysed) reader = FileTextReader(artifact.location) logger.debug('Looking for headers in: %s', reader.filename) for line in reader.line_by_line(): include_match: Optional[Match] \ = self._include_pattern.match(line) if include_match: include: str = include_match.group(1) logger.debug('Found header: %s', include) if include.startswith(('"', "'")): include = include.strip('"').strip("'") logger.debug(' * User header; adding dependency') new_artifact.add_dependency(Path(self._workspace / include)) return [new_artifact]
def run(self, artifacts: List[Artifact]) -> List[Artifact]: if len(artifacts) == 1: artifact = artifacts[0] else: msg = ('C Analyser expects only one Artifact, ' f'but was given {len(artifacts)}') raise TaskException(msg) reader = FileTextReader(artifact.location) state = CWorkingState(self.database) state.remove_c_file(reader.filename) new_artifact = Artifact(artifact.location, artifact.filetype, Analysed) state = CWorkingState(self.database) state.remove_c_file(reader.filename) index = clang.cindex.Index.create() translation_unit = index.parse(reader.filename, args=["-xc"]) # Create include region line mappings self._locate_include_regions(translation_unit) # Now walk the actual nodes and find all relevant external symbols usr_includes = [] current_def = None for node in translation_unit.cursor.walk_preorder(): if node.kind == clang.cindex.CursorKind.FUNCTION_DECL: if (node.is_definition() and node.linkage == clang.cindex.LinkageKind.EXTERNAL): # This should catch function definitions which are exposed # to the rest of the application current_def = CSymbolID(node.spelling, artifact.location) state.add_c_symbol(current_def) new_artifact.add_definition(node.spelling) else: # Any other declarations should be coming in via headers, # we can use the injected pragmas to work out whether these # are coming from system headers or user headers if (self._check_for_include( node.location.line) == "usr_include"): usr_includes.append(node.spelling) elif (node.kind == clang.cindex.CursorKind.CALL_EXPR): # When encountering a function call we should be able to # cross-reference it with a definition seen earlier; and # if it came from a user supplied header then we will # consider it a dependency within the project if node.spelling in usr_includes and current_def is not None: # TODO: Assumption that the most recent exposed # definition encountered above is the one which # should lodge this dependency - is that true? state.add_c_dependency(current_def, node.spelling) new_artifact.add_dependency(node.spelling) return [new_artifact]
def _process_use_statement(self, analysed_file, obj): use_name = _typed_child(obj, Name) if not use_name: raise TaskException("ERROR finding name in use statement:", obj.string) use_name = use_name.string if use_name in self.ignore_mod_deps: logger.debug(f"ignoring use of {use_name}") elif use_name not in self._intrinsic_modules: # found a dependency on fortran analysed_file.add_module_dep(use_name)
def run(self, artifacts: List[Artifact]) -> List[Artifact]: if len(artifacts) == 1: artifact = artifacts[0] else: msg = ('Fortran Preprocessor expects only one Artifact, ' f'but was given {len(artifacts)}') raise TaskException(msg) command = [self._preprocessor] command.extend(self._flags) command.append(str(artifact.location)) output_file = (self._workspace / artifact.location.with_suffix('.f90').name) command.append(str(output_file)) subprocess.run(command, check=True) return [Artifact(output_file, artifact.filetype, Raw)]
def run(self, artifacts: List[Artifact]) -> List[Artifact]: if len(artifacts) < 1: msg = ('Linker expects at least one Artifact, ' f'but was given {len(artifacts)}') raise TaskException(msg) command = [self._linker] output_file = self._workspace / self._output_filename command.extend(['-o', str(output_file)]) for artifact in artifacts: command.append(str(artifact.location)) command.extend(self._flags) subprocess.run(command, check=True) return [Artifact(output_file, Executable, Linked)]
def run(self, artifacts: List[Artifact]) -> List[Artifact]: if len(artifacts) == 1: artifact = artifacts[0] else: msg = ('Header Analyser expects only one Artifact, ' f'but was given {len(artifacts)}') raise TaskException(msg) new_artifact = Artifact(artifact.location, artifact.filetype, HeadersAnalysed) reader = FileTextReader(artifact.location) for line in reader.line_by_line(): include_match: Optional[Match] \ = self._include_pattern.match(line) if include_match: include: str = include_match.group(1) if include.startswith(('"', "'")): include = include.strip('"').strip("'") new_artifact.add_dependency(Path(self._workspace / include)) return [new_artifact]
def run(self, artifacts: List[Artifact]) -> List[Artifact]: logger = logging.getLogger(__name__) if len(artifacts) == 1: artifact = artifacts[0] else: msg = ('Fortran Analyser expects only one Artifact, ' f'but was given {len(artifacts)}') raise TaskException(msg) reader = FileTextReader(artifact.location) new_artifact = Artifact(artifact.location, artifact.filetype, Analysed) state = FortranWorkingState(self.database) state.remove_fortran_file(reader.filename) normalised_source = FortranNormaliser(reader) scope: List[Tuple[str, str]] = [] for line in normalised_source.line_by_line(): logger.debug(scope) logger.debug('Considering: %s', line) if len(scope) == 0: unit_match: Optional[Match] \ = self._program_unit_pattern.match(line) if unit_match: unit_type: str = unit_match.group(1).lower() unit_name: str = unit_match.group(2).lower() logger.debug('Found %s called "%s"', unit_type, unit_name) unit_id = FortranUnitID(unit_name, reader.filename) state.add_fortran_program_unit(unit_id) new_artifact.add_definition(unit_name) scope.append((unit_type, unit_name)) continue use_match: Optional[Match] \ = self._use_pattern.match(line) if use_match: use_name: str = use_match.group(3).lower() if use_name in self._intrinsic_modules: logger.debug('Ignoring intrinsic module "%s"', use_name) else: if len(scope) == 0: use_message \ = '"use" statement found outside program unit' raise TaskException(use_message) logger.debug('Found usage of "%s"', use_name) unit_id = FortranUnitID(scope[0][1], reader.filename) state.add_fortran_dependency(unit_id, use_name) new_artifact.add_dependency(use_name) continue block_match: Optional[Match] = self._scoping_pattern.match(line) if block_match: # Beware we want the value of a different group to the one we # check the presence of. # block_name: str = block_match.group(1) \ and block_match.group(2).lower() block_nature: str = block_match.group(3).lower() logger.debug('Found %s called "%s"', block_nature, block_name) scope.append((block_nature, block_name)) continue proc_match: Optional[Match] \ = self._procedure_pattern.match(line) if proc_match: proc_nature = proc_match.group(1).lower() proc_name = proc_match.group(2).lower() logger.debug('Found %s called "%s"', proc_nature, proc_name) # Note: We append a tuple so double brackets. scope.append((proc_nature, proc_name)) continue iface_match: Optional[Match] = self._interface_pattern.match(line) if iface_match: iface_name = iface_match.group(1) \ and iface_match.group(1).lower() logger.debug('Found interface called "%s"', iface_name) scope.append(('interface', iface_name)) continue type_match: Optional[Match] = self._type_pattern.match(line) if type_match: type_name = type_match.group(3).lower() logger.debug('Found type called "%s"', type_name) scope.append(('type', type_name)) continue end_match: Optional[Match] = self._end_block_pattern.match(line) if end_match: end_nature: str = end_match.group(1) \ and end_match.group(1).lower() end_name: str = end_match.group(2) \ and end_match.group(2).lower() logger.debug('Found end of %s called %s', end_nature, end_name) exp: Tuple[str, str] = scope.pop() if end_nature is not None: if end_nature != exp[0]: end_message = 'Expected end of {exp} "{name}" ' \ 'but found {found}' end_values = { 'exp': exp[0], 'name': exp[1], 'found': end_nature } raise TaskException(end_message.format(**end_values)) if end_name is not None: if end_name != exp[1]: end_message = 'Expected end of {exp} "{name}" ' \ 'but found end of {found}' end_values = { 'exp': exp[0], 'name': exp[1], 'found': end_name } raise TaskException(end_message.format(**end_values)) return [new_artifact]
def run(self, artifacts: List[Artifact]) -> List[Artifact]: logger = logging.getLogger(__name__) if len(artifacts) == 1: artifact = artifacts[0] else: msg = ('Fortran Analyser expects only one Artifact, ' f'but was given {len(artifacts)}') raise TaskException(msg) reader = FileTextReader(artifact.location) new_artifact = Artifact(artifact.location, artifact.filetype, Analysed) state = FortranWorkingState(self.database) state.remove_fortran_file(reader.filename) logger.debug('Analysing: %s', reader.filename) # If this file defines any C symbol bindings it may also # end up with an entry in the C part of the database cstate = CWorkingState(self.database) cstate.remove_c_file(reader.filename) normalised_source = FortranNormaliser(reader) scope: List[Tuple[str, str]] = [] for line in normalised_source.line_by_line(): logger.debug(scope) logger.debug('Considering: %s', line) if len(scope) == 0: unit_match: Optional[Match] \ = self._program_unit_pattern.match(line) if unit_match is not None: unit_type: str = unit_match.group(1).lower() unit_name: str = unit_match.group(2).lower() logger.debug('Found %s called "%s"', unit_type, unit_name) unit_id = FortranUnitID(unit_name, reader.filename) state.add_fortran_program_unit(unit_id) new_artifact.add_definition(unit_name) scope.append((unit_type, unit_name)) continue use_match: Optional[Match] \ = self._use_pattern.match(line) if use_match is not None: use_name: str = use_match.group(3).lower() if use_name in self._intrinsic_modules: logger.debug('Ignoring intrinsic module "%s"', use_name) else: if len(scope) == 0: use_message \ = '"use" statement found outside program unit' raise TaskException(use_message) logger.debug('Found usage of "%s"', use_name) unit_id = FortranUnitID(scope[0][1], reader.filename) state.add_fortran_dependency(unit_id, use_name) new_artifact.add_dependency(use_name) continue block_match: Optional[Match] = self._scoping_pattern.match(line) if block_match is not None: # Beware we want the value of a different group to the one we # check the presence of. # block_name: str = block_match.group(1) \ and block_match.group(2).lower() block_nature: str = block_match.group(3).lower() logger.debug('Found %s called "%s"', block_nature, block_name) scope.append((block_nature, block_name)) continue proc_match: Optional[Match] \ = self._procedure_pattern.match(line) if proc_match is not None: proc_nature = proc_match.group(1).lower() proc_name = proc_match.group(2).lower() logger.debug('Found %s called "%s"', proc_nature, proc_name) scope.append((proc_nature, proc_name)) # Check for the procedure being symbol-bound to C cbind_match: Optional[Match] \ = self._cbind_pattern.match(line) if cbind_match is not None: cbind_name = cbind_match.group(2) # The name keyword on the bind statement is optional. # If it doesn't exist, the procedure name is used if cbind_name is None: cbind_name = proc_name cbind_name = cbind_name.lower().strip("'\"") logger.debug('Bound to C symbol "%s"', cbind_name) # A bind within an interface block means this is # exposure of a C-defined function to Fortran, # otherwise it is going the other way (allowing C # code to call the Fortran procedure) if any([stype == "interface" for stype, _ in scope]): # TODO: This is sort of hijacking the mechanism used # for Fortran module dependencies, only using the # symbol name. Longer term we probably need a more # elegant solution logger.debug('In an interface block; so a dependency') unit_id = FortranUnitID(scope[0][1], reader.filename) state.add_fortran_dependency(unit_id, cbind_name) new_artifact.add_dependency(cbind_name) else: # Add to the C database logger.debug('Not an interface block; so a definition') symbol_id = CSymbolID(cbind_name, reader.filename) cstate.add_c_symbol(symbol_id) new_artifact.add_definition(cbind_name) continue cbind_match = self._cbind_pattern.match(line) if cbind_match is not None: # This should be a line binding from C to a variable definition # (procedure binds are dealt with above) cbind_name = cbind_match.group(2) # The name keyword on the bind statement is optional. # If it doesn't exist, the Fortran variable name is used if cbind_name is None: var_search = re.search(r'.*::\s*(\w+)', line) if var_search: cbind_name = var_search.group(1) else: cbind_message \ = 'failed to find variable name ' \ 'on C bound variable' raise TaskException(cbind_message) cbind_name = cbind_name.lower().strip("'\"") logger.debug('Found C bound variable called "%s"', cbind_name) # Add to the C database symbol_id = CSymbolID(cbind_name, reader.filename) cstate.add_c_symbol(symbol_id) new_artifact.add_definition(cbind_name) iface_match: Optional[Match] = self._interface_pattern.match(line) if iface_match is not None: iface_name = iface_match.group(1) \ and iface_match.group(1).lower() logger.debug('Found interface called "%s"', iface_name) scope.append(('interface', iface_name)) continue type_match: Optional[Match] = self._type_pattern.match(line) if type_match is not None: type_name = type_match.group(3).lower() logger.debug('Found type called "%s"', type_name) scope.append(('type', type_name)) continue end_match: Optional[Match] = self._end_block_pattern.match(line) if end_match is not None: end_nature: str = end_match.group(1) \ and end_match.group(1).lower() end_name: str = end_match.group(2) \ and end_match.group(2).lower() logger.debug('Found end of %s called %s', end_nature, end_name) exp: Tuple[str, str] = scope.pop() if end_nature is not None: if end_nature != exp[0]: end_message = 'Expected end of {exp} "{name}" ' \ 'but found {found}' end_values = { 'exp': exp[0], 'name': exp[1], 'found': end_nature } raise TaskException(end_message.format(**end_values)) if end_name is not None: if end_name != exp[1]: end_message = 'Expected end of {exp} "{name}" ' \ 'but found end of {found}' end_values = { 'exp': exp[0], 'name': exp[1], 'found': end_name } raise TaskException(end_message.format(**end_values)) return [new_artifact]