def run(self):
"""
Execute the build steps in order.
This function also records metrics and creates a summary, including charts if matplotlib is installed.
The metrics can be found in the project workspace.
"""
start_time = datetime.now().replace(microsecond=0)
(self.project_workspace / BUILD_OUTPUT).mkdir(parents=True,
exist_ok=True)
self._init_logging()
init_metrics(metrics_folder=self.metrics_folder)
artefact_store = dict()
try:
with TimerLogger(f'running {self.project_label} build steps'
) as steps_timer:
for step in self.steps:
with TimerLogger(step.name) as step_timer:
step.run(artefact_store=artefact_store, config=self)
send_metric('steps', step.name, step_timer.taken)
except Exception as err:
logger.error(f'\n\nError running build steps:\n{err}')
raise Exception(f'\n\nError running build steps:\n{err}')
finally:
self._finalise_metrics(start_time, steps_timer)
self._finalise_logging()
def _analyse_source_code(self, artefact_store) -> Set[AnalysedFile]:
"""
Find the symbol defs and deps in each file.
This is slow so we record our progress as we go.
"""
# get a list of all the files we want to analyse
files: List[Path] = self.source_getter(artefact_store)
# take hashes of all the files we want to analyse
with TimerLogger(f"generating {len(files)} file hashes"):
file_hashes = self._get_file_checksums(files)
with TimerLogger("loading previous analysis results"):
prev_results = self._load_analysis_results(
latest_file_hashes=file_hashes)
changed, unchanged = self._what_needs_reanalysing(
prev_results=prev_results, latest_file_hashes=file_hashes)
with TimerLogger("analysing files"):
with self._new_analysis_file(unchanged) as csv_writer:
freshly_analysed_fortran, freshly_analysed_c = self._parse_files(
changed, csv_writer)
return unchanged | freshly_analysed_fortran | freshly_analysed_c
def run(self, artefact_store: Dict, config):
"""
Creates the *build_trees* artefact from the files in `self.source_getter`.
Does the following, in order:
- Create a hash of every source file. Used to check if it's already been analysed.
- Parse the C and Fortran files to find external symbol definitions and dependencies in each file.
- Analysis results are stored in a csv as-we-go, so analysis can be resumed if interrupted.
- Create a 'symbol table' recording which file each symbol is in.
- Work out the file dependencies from the symbol dependencies.
- At this point we have a source tree for the entire source.
- (Optionally) Extract a sub tree for every root symbol, if provided. For building executables.
This step uses multiprocessing, unless disabled in the :class:`~fab.steps.Step` class.
:param artefact_store:
Contains artefacts created by previous Steps, and where we add our new artefacts.
This is where the given :class:`~fab.artefacts.ArtefactsGetter` finds the artefacts to process.
:param config:
The :class:`fab.build_config.BuildConfig` object where we can read settings
such as the project workspace folder or the multiprocessing flag.
"""
super().run(artefact_store, config)
analysed_files = self._analyse_source_code(artefact_store)
# add special measure symbols for files which could not be parsed
if self.special_measure_analysis_results:
warnings.warn(
"SPECIAL MEASURE: injecting user-defined analysis results")
analysed_files.update(set(self.special_measure_analysis_results))
project_source_tree, symbols = self._analyse_dependencies(
analysed_files)
# add the file dependencies for MO FCM's "DEPENDS ON:" commented file deps (being removed soon)
with TimerLogger(
"adding MO FCM 'DEPENDS ON:' file dependency comments"):
add_mo_commented_file_deps(project_source_tree)
logger.info(f"source tree size {len(project_source_tree)}")
# build tree extraction for executables.
if self.root_symbols:
build_trees = self._extract_build_trees(project_source_tree,
symbols)
else:
build_trees = {None: project_source_tree}
# throw in any extra source we need, which Fab can't automatically detect (i.e. not using use statements)
for build_tree in build_trees.values():
self._add_unreferenced_deps(symbols, project_source_tree,
build_tree)
validate_dependencies(build_tree)
artefact_store[BUILD_TREES] = build_trees
def _parse_files(self, to_analyse: Iterable[HashedFile], analysis_dict_writer: csv.DictWriter) -> \
Tuple[Set[AnalysedFile], Set[AnalysedFile]]:
"""
Determine the symbols which are defined in, and used by, each file.
Returns the analysed_fortran and analysed_c as lists of :class:`~fab.dep_tree.AnalysedFile`
with no file dependencies, to be filled in later.
"""
# fortran
fortran_files = set(
filter(lambda f: f.fpath.suffix == '.f90', to_analyse))
with TimerLogger(
f"analysing {len(fortran_files)} preprocessed fortran files"):
analysed_fortran, fortran_exceptions = self._analyse_file_type(
fpaths=fortran_files,
analyser=self.fortran_analyser.run,
dict_writer=analysis_dict_writer)
# c
c_files = set(filter(lambda f: f.fpath.suffix == '.c', to_analyse))
with TimerLogger(f"analysing {len(c_files)} preprocessed c files"):
analysed_c, c_exceptions = self._analyse_file_type(
fpaths=c_files,
analyser=self.c_analyser.run,
dict_writer=analysis_dict_writer)
# errors?
all_exceptions = fortran_exceptions | c_exceptions
if all_exceptions:
logger.error(f"{len(all_exceptions)} analysis errors")
errs_str = "\n\n".join(map(str, all_exceptions))
logger.debug(f"\nSummary of analysis errors:\n{errs_str}")
# warn about naughty fortran usage?
if self.fortran_analyser.depends_on_comment_found:
warnings.warn(
"not recommended 'DEPENDS ON:' comment found in fortran code")
return analysed_fortran, analysed_c
def _analyse_dependencies(self, analysed_files: Iterable[AnalysedFile]):
"""
Turn symbol deps into file deps and build a source dependency tree for the entire source.
"""
with TimerLogger(
"converting symbol dependencies to file dependencies"):
# map symbols to the files they're in
symbols: Dict[str, Path] = self._gen_symbol_table(analysed_files)
# fill in the file deps attribute in the analysed file objects
self._gen_file_deps(analysed_files, symbols)
source_tree: Dict[Path,
AnalysedFile] = {a.fpath: a
for a in analysed_files}
return source_tree, symbols
def _gen_file_deps(self, analysed_files: Iterable[AnalysedFile],
symbols: Dict[str, Path]):
"""
Use the symbol table to convert symbol dependencies into file dependencies.
"""
deps_not_found = set()
with TimerLogger("converting symbol to file deps"):
for analysed_file in analysed_files:
for symbol_dep in analysed_file.symbol_deps:
file_dep = symbols.get(symbol_dep)
if not file_dep:
deps_not_found.add(symbol_dep)
logger.debug(
f"not found {symbol_dep} for {analysed_file.fpath}"
)
continue
analysed_file.file_deps.add(file_dep)
if deps_not_found:
logger.info(f"{len(deps_not_found)} deps not found")
def _new_analysis_file(self, unchanged: Iterable[AnalysedFile]):
"""
Create the analysis file from scratch, containing any content from its previous version which is still valid.
The returned context is a csv.DictWriter.
"""
with TimerLogger("starting analysis progress file"):
analysis_progress_file = open(
self._config.project_workspace / "__analysis.csv", "wt")
analysis_dict_writer = csv.DictWriter(
analysis_progress_file, fieldnames=AnalysedFile.field_names())
analysis_dict_writer.writeheader()
# re-write the progress so far
unchanged_rows = (af.to_str_dict() for af in unchanged)
analysis_dict_writer.writerows(unchanged_rows)
analysis_progress_file.flush()
yield analysis_dict_writer
analysis_progress_file.close()
def _extract_build_trees(self, project_source_tree, symbols):
"""
Find the subset of files needed to build each root symbol (executable).
Assumes we have been given a root symbol(s) or we wouldn't have been called.
Returns a build tree for every root symbol.
"""
build_trees = {}
for root in self.root_symbols:
with TimerLogger(f"extracting build tree for root '{root}'"):
build_tree = extract_sub_tree(project_source_tree,
symbols[root],
verbose=False)
logger.info(
f"target source tree size {len(build_tree)} (target '{symbols[root]}')"
)
build_trees[root] = build_tree
return build_trees