def test_Standard(self):
fti = FilenameTypeInfo()
self.assertEqual(fti.Desc, "Filename")
self.assertEqual(fti.ConstraintsDesc, "Value must be a valid file")
self.assertEqual(fti.ExpectedType, six.string_types)
self.assertTrue(fti.IsValidItem(_script_fullpath))
self.assertFalse(fti.IsValidItem(_script_dir))
self.assertFalse(fti.IsValidItem("filename that doesn't exist"))
def test_NoEnsureExists(self):
fti = FilenameTypeInfo(ensure_exists=False)
self.assertEqual(fti.Desc, "Filename")
self.assertEqual(fti.ConstraintsDesc, '')
self.assertEqual(fti.ExpectedType, six.string_types)
self.assertTrue(fti.IsValidItem(_script_fullpath))
self.assertTrue(fti.IsValidItem(_script_dir))
self.assertTrue(fti.IsValidItem("filename that doesn't exist"))
class CodeGenerator(
AtomicInputProcessingMixin,
ConditionalInvocationQueryMixin,
MultipleOutputMixin,
CodeGeneratorMod.CodeGenerator,
):
# ----------------------------------------------------------------------
# |
# | Public Properties
# |
# ----------------------------------------------------------------------
Name = DerivedProperty(name)
Description = DerivedProperty(description)
InputTypeInfo = DerivedProperty(
FilenameTypeInfo(
validation_expression=filename_validation_expression))
OriginalModuleFilename = calling_mod_filename
RequiresOutputName = requires_output_name
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
@staticmethod
@override
def IsSupportedContent(filename):
return is_supported_content_func is None or is_supported_content_func(
filename)
# ----------------------------------------------------------------------
# |
# | Protected Methods
# |
# ----------------------------------------------------------------------
@classmethod
@override
def _GetOptionalMetadata(cls):
return get_optional_metadata_func() + \
[ ( "plugin_settings", {} ),
] + \
super(CodeGenerator, cls)._GetOptionalMetadata()
# ----------------------------------------------------------------------
@classmethod
@override
def _GetRequiredMetadataNames(cls):
names = [
"plugin_name",
]
if cls.RequiresOutputName:
names += [
"output_name",
]
names += super(CodeGenerator, cls)._GetRequiredMetadataNames()
return names
# ----------------------------------------------------------------------
@classmethod
@override
def _CreateContext(cls, metadata, status_stream):
if metadata["plugin_name"] not in plugin_map:
raise CommandLine.UsageException(
"'{}' is not a valid plugin".format(
metadata["plugin_name"]))
plugin = plugin_map[metadata["plugin_name"]].Plugin
# Ensure that all plugin settings are present and that they
# are the expected type.
custom_settings = OrderedDict([
(k, v) for k, v in plugin.GenerateCustomSettingsAndDefaults()
])
plugin_settings = metadata["plugin_settings"]
for k, v in six.iteritems(plugin_settings):
if k not in custom_settings:
raise CommandLine.UsageException(
"'{}' is not a valid plugin setting".format(k))
desired_type = type(custom_settings[k])
if type(v) != desired_type:
assert isinstance(v,
(str, UnicodeDecodeError)), (v, type(v))
plugin_settings[k] = StringSerialization.DeserializeItem(
CreateFromPythonType(desired_type), v)
for k, v in six.iteritems(custom_settings):
if k not in plugin_settings:
plugin_settings[k] = v
metadata["plugin_settings"] = plugin.PreprocessMetadata(
plugin_settings)
# Invoke custom functionality
context = create_context_func(metadata, plugin)
context = plugin.PreprocessContext(context)
context["output_filenames"] = [
os.path.join(context["output_dir"], filename)
for filename in plugin.GenerateOutputFilenames(context)
]
context = plugin.PostprocessContext(context)
if postprocess_context_func:
context = postprocess_context_func(context, plugin)
return super(CodeGenerator,
cls)._CreateContext(context, status_stream)
# ----------------------------------------------------------------------
@classmethod
@override
def _InvokeImpl(
cls,
invoke_reason,
context,
status_stream,
verbose_stream,
verbose,
):
return invoke_func(
cls,
invoke_reason,
context,
status_stream,
verbose_stream,
verbose,
plugin_map[context["plugin_name"]].Plugin,
)
# ----------------------------------------------------------------------
@classmethod
@override
def _GetAdditionalGeneratorItems(cls, context):
# ----------------------------------------------------------------------
def ProcessorGeneratorItem(item):
if isinstance(item, six.string_types) and item in plugin_map:
return plugin_map[item].Plugin
return item
# ----------------------------------------------------------------------
plugin = plugin_map[context["plugin_name"]].Plugin
return [ cls,
cls.OriginalModuleFilename,
plugin,
] + \
list(plugin.GetAdditionalGeneratorItems(context)) + \
super(CodeGenerator, cls)._GetAdditionalGeneratorItems(context)
class Verifier(VerifierMod.Verifier):
"""Verifies Python source code using PyLint"""
# ----------------------------------------------------------------------
# |
# | Public Properties
# |
# ----------------------------------------------------------------------
Name = "PyLint"
Description = "Statically analyzes Python source code, reporting common mistakes and errors."
InputTypeInfo = FilenameTypeInfo(validation_expression=r".+?\.py")
DEFAULT_PASSING_SCORE = 9.0
# Environment variable name of a custom PyLint configuration file. A default
# configuration file will be used if this environment variable isn't defined.
CONFIGURATION_ENVIRONMENT_VAR_NAME = "DEVELOPMENT_ENVIRONMENT_PYTHON_VERIFIER_CONFIGURATION"
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
def __str__(self):
return CommonEnvironment.ObjectStrImpl(self, include_private=False)
# ----------------------------------------------------------------------
@staticmethod
def ItemToTestName(item_name, test_type_name):
dirname, basename = os.path.split(item_name)
name, ext = os.path.splitext(basename)
return os.path.join(
dirname, test_type_name, "{}_{}{}".format(
name,
inflect.singular_noun(test_type_name) or test_type_name,
ext,
))
# ----------------------------------------------------------------------
@staticmethod
def TestToItemName(test_filename):
dirname, basename = os.path.split(test_filename)
match = re.match(
r"^(?P<name>.+)_(?P<test_type>[^_\.]+Test)(?P<ext>\..+)$",
basename,
)
if not match:
raise Exception(
"'{}' is not a recognized test name".format(test_filename))
name = "{}{}".format(match.group("name"), match.group("ext"))
test_type = match.group("test_type")
if dirname and os.path.basename(dirname) == inflect.plural(test_type):
dirname = os.path.dirname(dirname)
return os.path.join(dirname, name)
# ----------------------------------------------------------------------
# |
# | Private Methods
# |
# ----------------------------------------------------------------------
@classmethod
def _GetOptionalMetadata(cls):
return [
("passing_score", None),
] + super(Verifier, cls)._GetOptionalMetadata()
# ----------------------------------------------------------------------
@classmethod
def _CreateContext(cls, metadata):
if metadata["passing_score"] is None:
metadata["passing_score"] = cls.DEFAULT_PASSING_SCORE
metadata["explicit_passing_score"] = False
else:
metadata["explicit_passing_score"] = True
return super(Verifier, cls)._CreateContext(metadata)
# ----------------------------------------------------------------------
@classmethod
def _InvokeImpl(
cls,
invoke_reason,
context,
status_stream,
verbose_stream,
verbose,
):
# If the file is being invoked as a test file, measure the file under test
# rather than the test itself.
filename = context["input"]
try:
filename = cls.TestToItemName(filename)
except:
pass
assert os.path.isfile(filename), filename
if os.path.basename(filename) == "__init__.py" and os.path.getsize(
filename) == 0:
return 0
# Create the lint file
configuration_file = os.getenv(
cls.CONFIGURATION_ENVIRONMENT_VAR_NAME) or os.path.join(
_script_dir, "PythonVerifier.default_configuration")
assert os.path.isfile(configuration_file), configuration_file
# Write the python script that invokes the linter
temp_filename = CurrentShell.CreateTempFilename(".py")
with open(temp_filename, 'w') as f:
f.write(
textwrap.dedent("""\
import sys
from pylint import lint
lint.Run([ r"--rcfile={config}",
r"--msg-template={{path}}({{line}}): [{{msg_id}}] {{msg}}",
r"{filename}",
])
""").format(
config=configuration_file,
filename=filename,
))
with CallOnExit(lambda: FileSystem.RemoveFile(temp_filename)):
# Run the generated file
command_line = 'python "{}"'.format(temp_filename)
sink = six.moves.StringIO()
output_stream = StreamDecorator([
sink,
verbose_stream,
])
regex_sink = six.moves.StringIO()
Process.Execute(command_line,
StreamDecorator([
regex_sink,
output_stream,
]))
regex_sink = regex_sink.getvalue()
result = 0
# Extract the results
match = re.search(
r"Your code has been rated at (?P<score>[-\d\.]+)/(?P<max>[\d\.]+)",
regex_sink,
re.MULTILINE,
)
if not match:
result = -1
else:
score = float(match.group("score"))
max_score = float(match.group("max"))
assert max_score != 0.0
# Don't measure scores for files in Impl directories
is_impl_file = os.path.basename(filename).endswith("Impl")
if is_impl_file and not context["explicit_passing_score"]:
passing_score = None
else:
passing_score = context["passing_score"]
output_stream.write(
textwrap.dedent("""\
Score: {score} (out of {max_score})
Passing Score: {passing_score}{explicit}
""").format(
score=score,
max_score=max_score,
passing_score=passing_score,
explicit=" (explicitly provided)"
if context["explicit_passing_score"] else '',
))
if passing_score is not None and score < passing_score:
result = -1
if result != 0 and not verbose:
status_stream.write(sink.getvalue())
return result
class CodeGenerator(
AtomicInputProcessingMixin,
ConditionalInvocationQueryMixin,
MultipleOutputMixin,
CodeGeneratorBase,
):
# ----------------------------------------------------------------------
# |
# | Properties
# |
# ----------------------------------------------------------------------
Name = Interface.DerivedProperty("Interface Compiler")
Description = Interface.DerivedProperty(
"Extracts C++ methods and generates code according to the specified plugin"
)
InputTypeInfo = Interface.DerivedProperty(
FilenameTypeInfo(
validation_expression=r".+(?:\.c|\.cpp|\.cc|\.h|\.hpp)", ), )
# ----------------------------------------------------------------------
# |
# | Private Methods
# |
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def _GetRequiredMetadataNames(cls):
return [
"plugin_name",
"output_name",
] + super(CodeGenerator, cls)._GetRequiredMetadataNames()
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def _CreateContext(cls, metadata, status_stream):
# Ensure that all plugin settings are present and that they are the
# expected type.
custom_settings = OrderedDict([
(k, v)
for k, v in plugin.GenerateCustomMetadataSettingsAndDefaults()
])
assert "plugin_settings" in metadata
plugin_settings = metadata["plugin_settings"]
for k, v in six.iteritems(plugin_settings):
if k not in custom_settings:
raise Exception(
"'{}' is not a valid plugin setting".format(k))
desired_type = type(custom_settings[k])
if type(
v
) != desired_type: # <prefer isinstance> pylint: disable = C0123
assert isinstance(v, six.string_types), (v, type(v))
plugin_settings[k] = StringSerialization.DeserializeItem(
CreateFromPythonType(desired_type), v)
for k, v in six.iteritems(custom_settings):
if k not in plugin_settings:
plugin_settings[k] = v
# Ensure that the required metadata is present
for required in plugin.GetRequiredMetadataNames():
if required not in plugin_settings:
raise Exception(
"'{}' is required but was not found".format(required))
metadata["plugin_settings"] = plugin.PreprocessMetadata(
plugin_settings)
context = plugin.PreprocessContext(metadata)
# Create data based on the input files
context["plugin_context"] = ExtractContent(
context["inputs"],
status_stream,
)
context["output_filenames"] = [
os.path.join(context["output_dir"], filename)
for filename in plugin.GenerateOutputFilenames(context)
]
context = plugin.PostprocessContext(context)
del context["include_regexes"]
del context["exclude_regexes"]
return super(CodeGenerator,
cls)._CreateContext(context, status_stream)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def _GetAdditionalGeneratorItems(cls, context):
return [plugin] + plugin.GetAdditionalGeneratorFilenames() + super(
CodeGenerator, cls)._GetAdditionalGeneratorItems(context)
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def _InvokeImpl(
invoke_reason,
context,
status_stream,
verbose_stream,
verbose,
):
return plugin.Execute(invoke_reason, context, status_stream,
verbose_stream, verbose)
class DistutilsCompilerImpl(
AtomicInputProcessingMixin,
AlwaysInvocationQueryMixin,
MultipleOutputMixin,
CompilerImpl,
):
"""
Many different python compilers (such as cx_Freexe, Py2Exe, etc.) rely on
distutils functionality. This base class can be used to implement many of
the common details.
"""
# ----------------------------------------------------------------------
# | Public Properties
Description = "Creates an executable for a python file."
InputTypeInfo = FilenameTypeInfo(validation_expression=r".+\.py")
(
BuildType_Console,
BuildType_Windows,
) = range(2)
# ----------------------------------------------------------------------
# | Public Methods
@classmethod
def _GetRequiredContextNames(cls):
return [ "output_dir", ] + \
super(DistutilsCompilerImpl, cls)._GetRequiredContextNames()
# ----------------------------------------------------------------------
@classmethod
def _GetOptionalMetadata(cls):
return [
("preserve_temp_dir", False),
# General modifiers
("build_type", cls.BuildType_Console),
("include_tcl", False),
("no_optimize", False),
("no_bundle", False),
("manifest_filename", None),
("icon_filename", None),
("paths", []),
("includes", []),
("excludes", []),
("packages", []),
("distutil_args", []),
("output_name", None),
# Embedded metadata
("comments", ''),
("company_name", ''),
("file_description", ''),
("internal_name", ''),
("copyright", ''),
("trademark", ''),
("name", ''),
("version", ''),
] + super(DistutilsCompilerImpl, cls)._GetOptionalMetadata()
# ----------------------------------------------------------------------
@classmethod
def _CreateContext(cls, metadata):
if len(metadata["inputs"]) != 1 and metadata["output_name"]:
raise Exception(
"'output_name' cannot be specified when multiple input files are provided"
)
# Don't specify output filenames, as there will be many
metadata["output_filenames"] = []
# Ensure that the paths of all the inputs are included as search paths
for input_filename in metadata["inputs"]:
dirname = os.path.dirname(input_filename)
if dirname not in metadata["paths"]:
metadata["paths"].append(dirname)
if not metadata["include_tcl"]:
metadata["excludes"] += [
"tkconstants",
"tkinter",
"tcl",
]
del metadata["include_tcl"]
return super(DistutilsCompilerImpl, cls)._CreateContext(metadata)
# ----------------------------------------------------------------------
@classmethod
def _InvokeImpl(cls, invoke_reason, context, status_stream, verbose_stream,
verbose):
with status_stream.DoneManager(
associated_stream=verbose_stream) as (this_dm,
this_verbose_stream):
generated_python_context = cls._GenerateScriptContent(context)
assert generated_python_context
temp_filename = CurrentShell.CreateTempFilename(".py")
with open(temp_filename, 'w') as f:
f.write(generated_python_context)
if context["preserve_temp_dir"]:
this_dm.stream.write("Writing to '{}'\n".format(temp_filename))
cleanup_func = lambda: None
else:
cleanup_func = lambda: os.remove(temp_filename)
try:
sink = six.moves.StringIO()
this_dm.result = cls._Compile(
context, temp_filename,
StreamDecorator([
sink,
this_verbose_stream,
]))
if this_dm.result != 0:
if not verbose:
this_dm.stream.write(sink.getvalue())
return this_dm.result
finally:
if this_dm.result == 0:
cleanup_func()
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
@staticmethod
@abstractmethod
def _GenerateScriptContent(context):
"""
Returns python code in a string that is written to a temporary file which is then
passed to _Compile.
"""
raise Exception("Abstract method")
# ----------------------------------------------------------------------
@staticmethod
@abstractmethod
def _Compile(context, script_filename, output_stream):
"""Returns a result code"""
raise Exception("Abstract method")
class Formatter(FormatterImpl):
SUPPORTED_EXTENSIONS = [
".cpp",
".cc",
".c",
".hpp",
".h",
]
# ----------------------------------------------------------------------
# | Properties
Name = Interface.DerivedProperty("C++")
Description = Interface.DerivedProperty(
"Formats C++ code using clang-format plus enhancements")
InputTypeInfo = Interface.DerivedProperty(
FilenameTypeInfo(validation_expression=r".+(?:{})".format(
"|".join([re.escape(ext) for ext in SUPPORTED_EXTENSIONS]), ), ), )
# ----------------------------------------------------------------------
# | Methods
_is_initialized = False
@classmethod
def __clsinit__(cls, *plugin_input_dirs):
if cls._is_initialized:
return
plugins = cls._GetPlugins(
os.path.join(_script_dir, "CppFormatterImpl"),
*plugin_input_dirs,
)
debug_plugin = None
for potential_plugin in plugins:
if potential_plugin.Name == "Debug":
debug_plugin = potential_plugin
break
cls._plugins = plugins
cls._debug_plugin = debug_plugin
cls._is_initialized = True
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def Format(cls,
filename_or_content,
include_plugin_names=None,
exclude_plugin_names=None,
debug=False,
hint_filename=None,
*plugin_input_dirs,
**plugin_args):
cls.__clsinit__(*plugin_input_dirs)
if FileSystem.IsFilename(filename_or_content):
with open(filename_or_content) as f:
filename_or_content = f.read()
input_content = filename_or_content
del filename_or_content
include_plugin_names = set(include_plugin_names or [])
exclude_plugin_names = set(exclude_plugin_names or [])
if debug:
if include_plugin_names:
include_plugin_names.add(cls._debug_plugin.Name)
else:
exclude_plugin_names.add(cls._debug_plugin.Name)
plugins = [
plugin for plugin in cls._plugins
if plugin.Name not in exclude_plugin_names and
(not include_plugin_names or plugin.Name in include_plugin_names)
]
# Preprocess the content
lines = input_content.split("\n")
for plugin in plugins:
lines = plugin.PreprocessLines(lines)
input_content = "\n".join(lines)
# Invoke clang-format
command_line = "clang-format -style=file{}".format(
' "-assume-filename={}"'.format(hint_filename)
if hint_filename is not None else "", )
result, output = Process.Execute(
command_line,
stdin=input_content,
)
if result != 0:
raise Exception(
textwrap.dedent(
"""\
clang-format failed: {}
{}
""", ).format(result, StringHelpers.LeftJustify(output,
4)), )
# Convert the lines into line structures
lines = []
continuation_block_index = 0
continuation_block_id = None
for line_content in output.split("\n"):
has_continuation_token = False
if line_content.endswith("\\"):
if continuation_block_id is None:
continuation_block_id = continuation_block_index
continuation_block_index += 1
has_continuation_token = True
line_content = line_content[:-1].rstrip()
line = PluginBase.Line(
line_content,
continuation_block_id=continuation_block_id,
)
if not has_continuation_token:
continuation_block_id = None
lines.append(line)
# Decorate the lines
for plugin in plugins:
args = []
kwargs = {}
defaults = plugin_args.get(plugin.Name, None)
if defaults is not None:
if isinstance(defaults, (list, tuple)):
args = defaults
elif isinstance(defaults, dict):
kwargs = defaults
else:
assert False, defaults
lines = plugin.Decorate(lines, *args, **kwargs)
# Postprocess the lines
for plugin in plugins:
lines = plugin.PostprocessLines(lines)
# Restore the line continuation chars
output = []
continuation_block_id = None
for line_index, line in enumerate(lines):
# Add continuation chars
if line.continuation_block_id is not None and line.continuation_block_id != continuation_block_id:
continuation_block_id = line.continuation_block_id
# Process all the lines in this block
max_line_length = 0
end_block_index = line_index
while end_block_index < len(lines) and lines[
end_block_index].continuation_block_id == continuation_block_id:
max_line_length = max(max_line_length,
len(lines[end_block_index].content))
end_block_index += 1
max_line_length += 2
index = line_index
while index < end_block_index:
if index + 1 == end_block_index:
suffix = ""
else:
suffix = "{}\\".format(
" " *
(max_line_length - len(lines[index].content)))
lines[index].content = "{}{}".format(
lines[index].content, suffix)
index += 1
output = "\n".join([line.content for line in lines])
return output, output != input_content