def __init__(self, config_file_path, report_path_override=None, verbosity=1, source_path_override=None):
"""
Initialize AnnotationConfig.
Args:
config_file_path: Path to the configuration file
report_path_override: Path to write reports to, if overridden on the command line
verbosity: Verbosity level from the command line
source_path_override: Path to search if we're static code searching, if overridden on the command line
"""
self.groups = {}
self.choices = {}
self.optional_groups = []
self.annotation_tokens = []
self.annotation_regexes = []
self.mgr = None
# Global logger, other objects can hold handles to this
self.echo = VerboseEcho()
with open(config_file_path) as config_file:
raw_config = yaml.safe_load(config_file)
self._check_raw_config_keys(raw_config)
self.safelist_path = raw_config['safelist_path']
self.extensions = raw_config['extensions']
self.verbosity = verbosity
self.echo.set_verbosity(verbosity)
self.report_path = report_path_override if report_path_override else raw_config['report_path']
self.echo(f"Configured for report path: {self.report_path}")
self.source_path = source_path_override if source_path_override else raw_config['source_path']
self.echo(f"Configured for source path: {self.source_path}")
self._configure_coverage(raw_config.get('coverage_target', None))
self.report_template_dir = raw_config.get('report_template_dir')
self.rendered_report_dir = raw_config.get('rendered_report_dir')
self.rendered_report_file_extension = raw_config.get('rendered_report_file_extension')
self.rendered_report_source_link_prefix = raw_config.get('rendered_report_source_link_prefix')
self._configure_annotations(raw_config)
self._configure_extensions()
def test_nothing_found():
"""
Make sure nothing fails when no annotation is found.
"""
config = FakeConfig()
r = FakeExtension(config, VerboseEcho())
with open('tests/extensions/base_test_files/empty.foo') as f:
r.search(f)
def test_multi_line_annotations(test_file, annotations):
config = AnnotationConfig('tests/test_configurations/.annotations_test')
annotator = PythonAnnotationExtension(config, VerboseEcho())
with open(f'tests/extensions/python_test_files/{test_file}') as fi:
result_annotations = annotator.search(fi)
assert len(annotations) == len(result_annotations)
for annotation, result_annotation in zip(annotations, result_annotations):
assert result_annotation['annotation_token'] == annotation[0]
assert result_annotation['annotation_data'] == annotation[1]
def test_strip_single_line_comment_tokens():
config = FakeConfig()
extension = FakeExtension(config, VerboseEcho())
text = """baz line1
baz line2
bazline3
baz line4"""
expected_result = """ line1
line2
line3
line4"""
# pylint: disable=protected-access
assert expected_result == extension._strip_single_line_comment_tokens(text)
class AnnotationConfig:
"""
Configuration shared among all Code Annotations commands.
"""
def __init__(self, config_file_path, report_path_override=None, verbosity=1, source_path_override=None):
"""
Initialize AnnotationConfig.
Args:
config_file_path: Path to the configuration file
report_path_override: Path to write reports to, if overridden on the command line
verbosity: Verbosity level from the command line
source_path_override: Path to search if we're static code searching, if overridden on the command line
"""
self.groups = {}
self.choices = {}
self.optional_groups = []
self.annotation_tokens = []
self.annotation_regexes = []
self.mgr = None
# Global logger, other objects can hold handles to this
self.echo = VerboseEcho()
with open(config_file_path) as config_file:
raw_config = yaml.safe_load(config_file)
self._check_raw_config_keys(raw_config)
self.safelist_path = raw_config['safelist_path']
self.extensions = raw_config['extensions']
self.verbosity = verbosity
self.echo.set_verbosity(verbosity)
self.report_path = report_path_override if report_path_override else raw_config['report_path']
self.echo(f"Configured for report path: {self.report_path}")
self.source_path = source_path_override if source_path_override else raw_config['source_path']
self.echo(f"Configured for source path: {self.source_path}")
self._configure_coverage(raw_config.get('coverage_target', None))
self.report_template_dir = raw_config.get('report_template_dir')
self.rendered_report_dir = raw_config.get('rendered_report_dir')
self.rendered_report_file_extension = raw_config.get('rendered_report_file_extension')
self.rendered_report_source_link_prefix = raw_config.get('rendered_report_source_link_prefix')
self._configure_annotations(raw_config)
self._configure_extensions()
def _check_raw_config_keys(self, raw_config):
"""
Validate that all required keys exist in the configuration file.
Args:
raw_config: Python representation of the YAML config file
Raises:
ConfigurationException on any missing keys
"""
errors = []
for k in ('report_path', 'source_path', 'safelist_path', 'annotations', 'extensions'):
if k not in raw_config:
errors.append(k)
if errors:
raise ConfigurationException(
'The following required keys are missing from the configuration file: \n{}'.format(
'\n'.join(errors)
)
)
def _is_annotation_group(self, token_or_group):
"""
Determine if an annotation is a group or not.
Args:
token_or_group: The annotation being checked
Returns:
True if the type of the annotation is correct for a group, otherwise False
"""
return isinstance(token_or_group, list)
def _is_choice_group(self, token_or_group):
"""
Determine if an annotation is a choice group.
Args:
token_or_group: The annotation being checked
Returns:
True if the type of the annotation is correct for a choice group, otherwise False
"""
return isinstance(token_or_group, dict) and "choices" in token_or_group
def _is_optional_group(self, token_or_group):
"""
Determine if an annotation is an optional group.
Args:
token_or_group: The annotation being checked
Returns:
True if the annotation is optional, otherwise False.
"""
return isinstance(token_or_group, dict) and bool(token_or_group.get("optional"))
def _is_annotation_token(self, token_or_group):
"""
Determine if an annotation has the right format.
Args:
token_or_group: The annotation being checked
Returns:
True if the type of the annotation is correct for a text type, otherwise False
"""
if token_or_group is None:
return True
if isinstance(token_or_group, dict):
# If annotation is a dict, only a few keys are tolerated
return set(token_or_group.keys()).issubset({"choices", "optional"})
return False
def _add_annotation_token(self, token):
if token in self.annotation_tokens:
raise ConfigurationException(f'{token} is configured more than once, tokens must be unique.')
self.annotation_tokens.append(token)
def _configure_coverage(self, coverage_target):
"""
Set coverage_target to the specified value.
Args:
coverage_target:
Returns:
"""
if coverage_target:
try:
self.coverage_target = float(coverage_target)
except (TypeError, ValueError) as error:
raise ConfigurationException(
f'Coverage target must be a number between 0 and 100 not "{coverage_target}".'
) from error
if self.coverage_target < 0.0 or self.coverage_target > 100.0:
raise ConfigurationException(
f'Invalid coverage target. {self.coverage_target} is not between 0 and 100.'
)
else:
self.coverage_target = None
def _configure_group(self, group_name, group):
"""
Perform group configuration and add annotations from the group to global configuration.
Args:
group_name: The name of the group (the key in the configuration dictionary)
group: The list of annotations that comprise the group
Raises:
TypeError if the group is misconfigured
"""
self.groups[group_name] = []
if not group or len(group) == 1:
raise ConfigurationException(f'Group "{group_name}" must have more than one annotation.')
for annotation in group:
for annotation_token in annotation:
annotation_value = annotation[annotation_token]
# Otherwise it should be a text type, if not then error out
if not self._is_annotation_token(annotation_value):
raise ConfigurationException(f'{annotation} is an unknown annotation type.')
# The annotation comment is a choice group
if self._is_choice_group(annotation_value):
self._configure_choices(annotation_token, annotation_value)
# The annotation comment is not mandatory
if self._is_optional_group(annotation_value):
self.optional_groups.append(annotation_token)
self.groups[group_name].append(annotation_token)
self._add_annotation_token(annotation_token)
self.annotation_regexes.append(re.escape(annotation_token))
def _configure_choices(self, annotation_token, annotation):
"""
Configure the choices list for an annotation.
Args:
annotation_token: The annotation token we are setting choices for
annotation: The annotation body (list of choices)
"""
self.choices[annotation_token] = annotation['choices']
def _configure_annotations(self, raw_config):
"""
Transform the configured annotations into more usable pieces and validate.
Args:
raw_config: The dictionary form of our configuration file
Raises:
TypeError if annotations are misconfigured
"""
annotation_tokens = raw_config['annotations']
for annotation_token_or_group_name in annotation_tokens:
annotation = annotation_tokens[annotation_token_or_group_name]
if self._is_annotation_group(annotation):
self._configure_group(annotation_token_or_group_name, annotation)
elif self._is_choice_group(annotation):
self._configure_choices(annotation_token_or_group_name, annotation)
self._add_annotation_token(annotation_token_or_group_name)
self.annotation_regexes.append(re.escape(annotation_token_or_group_name))
elif not self._is_annotation_token(annotation): # pragma: no cover
raise TypeError(
f'{annotation_token_or_group_name} is an unknown type, must be strings or lists.'
)
else:
self._add_annotation_token(annotation_token_or_group_name)
self.annotation_regexes.append(re.escape(annotation_token_or_group_name))
self.echo.echo_v(f"Groups configured: {self.groups}")
self.echo.echo_v(f"Choices configured: {self.choices}")
self.echo.echo_v(f"Annotation tokens configured: {self.annotation_tokens}")
def _plugin_load_failed_handler(self, *args, **kwargs):
"""
Handle failures to load an extension.
Dumps the error and raises an exception. By default these
errors just fail silently.
Args:
*args:
**kwargs:
Raises:
ConfigurationException
"""
self.echo(str(args), fg='red')
self.echo(str(kwargs), fg='red')
raise ConfigurationException('Failed to load a plugin, aborting.')
def _configure_extensions(self):
"""
Configure the Stevedore NamedExtensionManager.
Raises:
ConfigurationException
"""
# These are the names of all of our configured extensions
configured_extension_names = self.extensions.keys()
# Load Stevedore extensions that we are configured for (and only those)
self.mgr = named.NamedExtensionManager(
names=configured_extension_names,
namespace='annotation_finder.searchers',
invoke_on_load=True,
on_load_failure_callback=self._plugin_load_failed_handler,
invoke_args=(self, self.echo),
)
# Output extension names listed in configuration
self.echo.echo_vv("Configured extension names: {}".format(" ".join(configured_extension_names)))
# Output found extension entry points from setup.py|cfg (whether or not they were loaded)
self.echo.echo_vv("Stevedore entry points found: {}".format(str(self.mgr.list_entry_points())))
# Output extensions that were actually able to load
self.echo.echo_v("Loaded extensions: {}".format(" ".join([x.name for x in self.mgr.extensions])))
if len(self.mgr.extensions) != len(configured_extension_names):
raise ConfigurationException('Not all configured extensions could be loaded! Asked for {} got {}.'.format(
configured_extension_names, self.mgr.extensions
))