def _CreateContext(cls, metadata, status_stream):
if metadata["plugin_name"] not in plugin_map:
raise CommandLine.UsageException(
"'{}' is not a valid plugin".format(
metadata["plugin_name"]))
plugin = plugin_map[metadata["plugin_name"]].Plugin
# Ensure that all plugin settings are present and that they
# are the expected type.
custom_settings = OrderedDict([
(k, v) for k, v in plugin.GenerateCustomSettingsAndDefaults()
])
plugin_settings = metadata["plugin_settings"]
for k, v in six.iteritems(plugin_settings):
if k not in custom_settings:
raise CommandLine.UsageException(
"'{}' is not a valid plugin setting".format(k))
desired_type = type(custom_settings[k])
if type(v) != desired_type:
assert isinstance(v,
(str, UnicodeDecodeError)), (v, type(v))
plugin_settings[k] = StringSerialization.DeserializeItem(
CreateFromPythonType(desired_type), v)
for k, v in six.iteritems(custom_settings):
if k not in plugin_settings:
plugin_settings[k] = v
metadata["plugin_settings"] = plugin.PreprocessMetadata(
plugin_settings)
# Invoke custom functionality
context = create_context_func(metadata, plugin)
context = plugin.PreprocessContext(context)
context["output_filenames"] = [
os.path.join(context["output_dir"], filename)
for filename in plugin.GenerateOutputFilenames(context)
]
context = plugin.PostprocessContext(context)
if postprocess_context_func:
context = postprocess_context_func(context, plugin)
return super(CodeGenerator,
cls)._CreateContext(context, status_stream)
def ToRegex(value):
try:
return re.compile(value)
except:
raise CommandLine.UsageException(
"'{}' is not a valid regular expression".format(value),
)
def Create(
profile_name,
host,
username=None,
password=None,
port=26,
from_name=None,
from_email=None,
ssl=False,
output_stream=sys.stdout,
):
"""Creates a new SmtpMailer profile"""
if not from_name and not from_email:
raise CommandLine.UsageException(
"'from_name' or 'from_email' must be provided")
mailer = SmtpMailer(
host,
username=username,
password=password,
port=port,
from_name=from_name,
from_email=from_email,
ssl=ssl,
)
mailer.Save(profile_name)
output_stream.write(
"The profile '{}' has been created.\n".format(profile_name))
def Lcov(
bin_dir=None,
not_llvm=False,
output_dir=None,
output_filename="lcov.info",
type=None,
output_stream=sys.stdout,
verbose=False,
):
"""Generates a LCOV file based on *.gcno files"""
bin_dirs = bin_dir
del bin_dir
if not bin_dirs:
bin_dirs.append(os.getcwd())
if len(bin_dirs) > 1 and not output_dir:
raise CommandLine.UsageException(
"An 'output_dir' must be provided when multiple 'bin_dirs' are parsed",
)
if len(bin_dirs) == 1 and not output_dir:
output_dir = bin_dirs[0]
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
output_filename = os.path.join(output_dir, output_filename)
dm.stream.write("Creating '{}'...".format(output_filename))
with dm.stream.DoneManager() as this_dm:
FileSystem.MakeDirs(output_dir)
command_line = 'grcov {dirs} -o "{output_filename}"{llvm}{type}'.format(
dirs=" ".join(['"{}"'.format(dir) for dir in bin_dirs]),
output_filename=output_filename,
llvm="" if not_llvm else " --llvm",
type="" if type is None else " -t {}".format(type),
)
if verbose:
this_dm.stream.write(
textwrap.dedent(
"""\
Command Line:
{}
""",
).format(command_line),
)
this_dm.result = Process.Execute(command_line, this_dm.stream)
if this_dm.result != 0:
return this_dm.result
return dm.result
def _Impl(working_dir, output_stream, verbose, callback_func):
if not os.path.isfile(os.path.join(working_dir, "CMakeLists.txt")):
raise CommandLine.UsageException(
"The directory '{}' does not contain the file 'CMakeLists.txt'".format(
working_dir,
),
)
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
build_dir_prefix = [
"build",
CurrentShell.CategoryName,
os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_COMPILER_NAME"),
os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_ARCHITECTURE"),
]
configuration_types = ["Debug", "Release"]
# Tests cannot execute in parallel, as they must be invoked from the build
# dir (which impacts the global working directory)
test_lock = threading.Lock()
# ----------------------------------------------------------------------
def Impl(task_index, output_stream, on_status_update):
configuration = configuration_types[task_index]
build_dir = os.path.join(
*([working_dir] + build_dir_prefix + [configuration])
)
return callback_func(
test_lock,
configuration,
build_dir,
output_stream,
on_status_update,
)
# ----------------------------------------------------------------------
dm.result = TaskPool.Execute(
[
TaskPool.Task(configuration_type, Impl)
for configuration_type in configuration_types
],
dm.stream,
progress_bar=True,
verbose=verbose,
)
return dm.result
def CreateCleanMethod(compiler_type):
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints(
output_dir=CommandLine.DirectoryTypeInfo(),
output_stream=None,
)
def Clean(
output_dir,
output_stream=sys.stdout,
):
"""Cleans previously compiled output."""
return CommandLineClean(output_dir, output_stream)
# ----------------------------------------------------------------------
return Clean
def Build(
configuration,
output_dir,
release_build=False,
prerelease_build_name=None,
no_build_info=False,
keep_temp_dir=False,
cmake_generator=(
None if os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY_CONFIGURATION") == "universal_linux" or os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_USE_DEFAULT_CMAKE_GENERATOR") else "Ninja"
),
output_stream=sys.stdout,
verbose=False,
):
"""Builds the Featurizer Shared Library"""
if release_build and prerelease_build_name:
raise CommandLine.UsageException(
"A prerelese build name cannot be provided with the 'release_build' flag",
)
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
FileSystem.RemoveTree(output_dir)
FileSystem.MakeDirs(output_dir)
temp_directory = CurrentShell.CreateTempDirectory()
# ----------------------------------------------------------------------
def CleanupTempDir():
if keep_temp_dir:
dm.stream.write(
"\nCMake output has been written to '{}'.\n".format(temp_directory),
)
return
FileSystem.RemoveTree(temp_directory)
# ----------------------------------------------------------------------
with CallOnExit(CleanupTempDir):
prev_dir = os.getcwd()
os.chdir(temp_directory)
with CallOnExit(lambda: os.chdir(prev_dir)):
if not release_build:
if prerelease_build_name is None:
# This value should compare as:
# "manual" < "pipeline"
prerelease_build_name = "manual"
if not no_build_info:
now = datetime.datetime.now()
prerelease_build_name = "{prerelease_build_name}.{year}.{month}.{day}.{hour}.{minute}.{second}.{configuration}".format(
year=now.year,
month=now.month,
day=now.day,
hour=now.hour,
minute=now.minute,
second=now.second,
prerelease_build_name=prerelease_build_name,
configuration=configuration.lower(),
)
activities = [
(
"Generating cmake Files",
'cmake {generator}-DCMAKE_BUILD_TYPE={configuration} {prerelease_build_name} "{this_dir}"'.format(
generator='-G "{}" '.format(
cmake_generator,
) if cmake_generator else "",
temp_dir=temp_directory,
configuration=configuration,
this_dir=_script_dir,
prerelease_build_name="" if not prerelease_build_name else "-DPRODUCT_VERSION_PRERELEASE_INFO={}".format(
prerelease_build_name,
),
),
),
("Building", "cmake --build ."),
]
if (
os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY_CONFIGURATION")
== "universal_linux"
):
activities.append(
(
"Verifying Universal Linux Binaries",
"libcheck libFeaturizers.so",
),
)
activities += [
("Copying Binaries", _CopyBinaries),
("Copying Data", _CopyData),
("Copying Headers", _CopyHeaders),
]
for index, (activity, command_line) in enumerate(activities):
dm.stream.write(
"{} ({} of {})...".format(activity, index + 1, len(activities)),
)
with dm.stream.DoneManager(
suffix="\n" if verbose else None,
) as this_dm:
sink = six.moves.StringIO()
output_streams = [sink]
if verbose:
output_streams.append(
StreamDecorator(
this_dm.stream,
line_prefix="INFO: ",
),
)
this_output_stream = StreamDecorator(output_streams)
if callable(command_line):
this_dm.result = command_line(
temp_directory,
output_dir,
this_output_stream,
)
else:
this_dm.result = Process.Execute(
command_line,
this_output_stream,
)
if this_dm.result != 0:
if not verbose:
this_dm.stream.write(sink.getvalue())
return this_dm.result
return dm.result
from CommonEnvironment import Process
from CommonEnvironment.Shell.All import CurrentShell
from CommonEnvironment.StreamDecorator import StreamDecorator
# ----------------------------------------------------------------------
_script_fullpath = CommonEnvironment.ThisFullpath()
_script_dir, _script_name = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------
CONFIGURATIONS = ["Debug", "Release"]
JSON_FILENAME = "Microsoft MLFeaturizers.FileAttributes.json"
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints(
configuration=CommandLine.EnumTypeInfo(CONFIGURATIONS),
output_dir=CommandLine.DirectoryTypeInfo(
ensure_exists=False,
),
prerelease_build_name=CommandLine.StringTypeInfo(
arity="?",
),
cmake_generator=CommandLine.StringTypeInfo(
arity="?",
),
output_stream=None,
)
def Build(
configuration,
output_dir,
release_build=False,
from CommonEnvironment.CallOnExit import CallOnExit
from CommonEnvironment import CommandLine
from CommonEnvironment import FileSystem
from CommonEnvironment import Process
from CommonEnvironment.Shell.All import CurrentShell
# ----------------------------------------------------------------------
_script_fullpath = CommonEnvironment.ThisFullpath()
_script_dir, _script_name = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints(
arg=CommandLine.StringTypeInfo(
arity="+",
),
output_stream=None,
)
def EntryPoint(
arg,
output_stream=sys.stdout,
):
args = arg
del arg
# One of the args will be the filename
input_filename = None
for arg in args:
if arg.startswith("-assume-filename="):
import pandas as pd
import CommonEnvironment
from CommonEnvironment import CommandLine
from CommonEnvironment import FileSystem
from CommonEnvironment.StreamDecorator import StreamDecorator
# ----------------------------------------------------------------------
_script_fullpath = CommonEnvironment.ThisFullpath()
_script_dir, _script_name = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------
@CommandLine.EntryPoint()
@CommandLine.Constraints(
zipped_input_filename=CommandLine.FilenameTypeInfo(),
output_stream=None,
)
def EntryPoint(
zipped_input_filename,
output_stream=sys.stdout,
):
"""Generates JSON files based on data previously pickled"""
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
output_dir = os.path.join(_script_dir, "..", "GeneratedCode")
from CommonEnvironment import CommandLine
from CommonEnvironment import FileSystem
from CommonEnvironment import Process
from CommonEnvironment.Shell.All import CurrentShell
from CommonEnvironment.StreamDecorator import StreamDecorator
# ----------------------------------------------------------------------
_script_fullpath = CommonEnvironment.ThisFullpath()
_script_dir, _script_name = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints(
output_dir=CommandLine.DirectoryTypeInfo(ensure_exists=False),
output_stream=None,
)
def Build(
output_dir,
output_stream=sys.stdout,
):
paths = []
includes = []
excludes = []
command_line = '"{script}" Compile "/input={input}" "/output_dir={output_dir}" /no_bundle /no_optimize {paths}{includes}{excludes}' \
.format( script=CurrentShell.CreateScriptName("CxFreezeCompiler"),
input=os.path.join(_script_dir, "Backup.py"),
output_dir=output_dir,
paths='' if not paths else " {}".format(' '.join([ '"/path={}"'.format(path) for path in paths ])),
def _GenerateRebuild(build_func, clean_func, config):
assert build_func
assert clean_func
assert config
# ----------------------------------------------------------------------
def Impl(configuration, output_dir, build_func, clean_func):
result = clean_func(configuration, output_dir)
if result is not None and result != 0:
return result
if config.RequiresOutputDir and not os.path.isdir(output_dir):
os.makedirs(output_dir)
result = build_func(configuration, output_dir)
if result is not None and result != 0:
return result
return 0
# ----------------------------------------------------------------------
if config.Configurations:
if config.RequiresOutputDir:
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints( configuration=CommandLine.EnumTypeInfo(config.Configurations),
output_dir=CommandLine.DirectoryTypeInfo(ensure_exists=False),
)
def Rebuild(configuration, output_dir):
return Impl(configuration, output_dir, build_func, clean_func)
# ----------------------------------------------------------------------
else:
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints( configuration=CommandLine.EnumTypeInfo(config.Configurations),
)
def Rebuild(configuration):
return Impl( configuration,
None,
lambda cfg, output_dir: build_func(cfg),
lambda cfg, output_dir: clean_func(cfg),
)
# ----------------------------------------------------------------------
else:
if config.RequiresOutputDir:
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints( output_dir=CommandLine.DirectoryTypeInfo(ensure_exists=False),
)
def Rebuild(output_dir):
return Impl( None,
output_dir,
lambda cfg, output_dir: build_func(output_dir),
lambda cfg, output_dir: clean_func(output_dir),
)
# ----------------------------------------------------------------------
else:
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
def Rebuild():
return Impl( None,
None,
lambda cfg, output_dir: build_func(),
lambda cfg, output_dir: clean_func(),
)
# ----------------------------------------------------------------------
return Rebuild
# ----------------------------------------------------------------------
_script_fullpath = CommonEnvironment.ThisFullpath()
_script_dir, _script_name = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------
inflect = inflect_mod.engine()
# ----------------------------------------------------------------------
DOXYGEN_EXTENSION = ".doxygen"
DOXYGEN_EXTENSION_IGNORE = "{}-ignore".format(DOXYGEN_EXTENSION)
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints(
code_dir_or_doxygen_filename=CommandLine.FilenameTypeInfo(
match_any=True, ),
output_dir=CommandLine.DirectoryTypeInfo(ensure_exists=False, ),
output_stream=None,
)
def EntryPoint(
code_dir_or_doxygen_filename,
output_dir,
output_stream=sys.stdout,
verbose=False,
):
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
# Get the doxygen files
error_string = plugin_class.IsValidEnvironment()
if error_string is not None:
sys.stdout.write("INFO: {}\n".format(error_string))
continue
yield plugin_class
PLUGINS = OrderedDict([(plugin.Name, plugin) for plugin in EnumeratePlugins()])
del EnumeratePlugins
if not PLUGINS:
sys.stdout.write("\nWARNING: No plugins were found.\n")
sys.exit(1)
_PluginTypeInfo = CommandLine.EnumTypeInfo(list(six.iterkeys(PLUGINS)))
# ----------------------------------------------------------------------
SUPPORTED_TYPES = set([
"int8",
"int16",
"int32",
"int64",
"uint8",
"uint16",
"uint32",
"uint64",
"float",
"double",
"string",
"bool",
if context["explicit_passing_score"] else '',
))
if passing_score is not None and score < passing_score:
result = -1
if result != 0 and not verbose:
status_stream.write(sink.getvalue())
return result
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints(
input=CommandLine.FilenameTypeInfo(match_any=True, arity='+'),
passing_score=CommandLine.FloatTypeInfo(min=0.0, max=10.0, arity='?'),
output_stream=None,
)
def Verify(
input, # <Redefinig built-in type> pylint: disable = W0622
passing_score=None,
output_stream=sys.stdout,
verbose=False,
):
"""Verifies the given python input"""
inputs = input
del input
return VerifierMod.CommandLineVerify(
def Main( config,
original_args=sys.argv, # <Dangerous default value> pylint: disable = W0102
command_line_arg_prefix='/',
command_line_keyword_separator='=',
command_line_dict_tag_value_separator=':',
verbose=False,
output_stream=sys.stdout,
):
"""Method called in a build file's entry point"""
assert config
assert original_args
assert command_line_arg_prefix
assert command_line_keyword_separator
assert command_line_dict_tag_value_separator
assert output_stream
# Some build file functions are required, others are not
required = { "build" : lambda ep: _RedirectEntryPoint("Build", ep, config),
"clean" : lambda ep: _RedirectEntryPoint("Clean", ep, config),
}
required_names = set(required.keys())
entry_points = CommandLine.EntryPointInformation.FromModule(sys.modules["__main__"])
for entry_point in entry_points:
entry_point_name_lower = entry_point.Name.lower()
if entry_point_name_lower in required_names:
required[entry_point_name_lower](entry_point)
required[entry_point_name_lower] = entry_point
required_names.remove(entry_point_name_lower)
else:
for reserved_name in [ "Rebuild",
"Metadata",
]:
if entry_point_name_lower == reserved_name.lower():
raise Exception("The name '{}' is reserved and will be automatically generated".format(reserved_name))
if required_names:
raise Exception("These methods must be defined: {}".format(', '.join(required_names)))
entry_points.append(CommandLine.EntryPointInformation.FromFunction(_GenerateRebuild( required["build"],
required["clean"],
config,
)))
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
def Metadata():
sys.stdout.write(str(config))
# ----------------------------------------------------------------------
entry_points.append(CommandLine.EntryPointInformation.FromFunction(Metadata))
config = CompleteConfiguration( [ entry_point.Name for entry_point in entry_points ],
config,
)
script_description_suffix = None
if config.Configurations:
script_description_suffix = " Where <configuration> can be:\n\n{}\n".format('\n'.join([ " - {}".format(cfg) for cfg in config.Configurations ]))
# Execute
stack_frame = inspect.stack()[-1]
current_dir = os.getcwd()
os.chdir(os.path.dirname(os.path.abspath(stack_frame[1])))
with CallOnExit(lambda: os.chdir(current_dir)):
# Ensure that an output directory is created prior to invoking build functionality
if config.RequiresOutputDir and len(original_args) >= 2 and original_args[1].lower() in [ "build", "rebuild", ]:
output_dir = None
if config.Configurations:
# Command line is: <script> build <config> <output_dir> ...
if len(original_args) >= 4:
output_dir = original_args[3]
else:
# Command line is: <script build <output_dir>
if len(original_args) >= 3:
output_dir = original_args[2]
if output_dir:
FileSystem.MakeDirs(output_dir)
return CommandLine.Executor( args=original_args,
command_line_arg_prefix=command_line_arg_prefix,
command_line_keyword_separator=command_line_keyword_separator,
command_line_dict_tag_value_separator=command_line_dict_tag_value_separator,
script_description=inspect.getmodule(stack_frame[0]).__doc__ or '',
script_description_suffix=script_description_suffix,
entry_points=entry_points,
).Invoke( verbose=verbose,
output_stream=output_stream,
)
import CommonEnvironment
from CommonEnvironment import CommandLine
from CommonEnvironmentEx.SmtpMailer import SmtpMailer
# ----------------------------------------------------------------------
_script_fullpath = CommonEnvironment.ThisFullpath()
_script_dir, _script_name = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints(
profile_name=CommandLine.StringTypeInfo(),
host=CommandLine.StringTypeInfo(),
username=CommandLine.StringTypeInfo(arity='?'),
password=CommandLine.StringTypeInfo(arity='?'),
port=CommandLine.IntTypeInfo(min=1, arity='?'),
from_name=CommandLine.StringTypeInfo(arity='?'),
from_email=CommandLine.StringTypeInfo(arity='?'),
output_stream=None,
)
def Create(
profile_name,
host,
username=None,
password=None,
port=26,
from_name=None,
def CreateCompileMethod(compiler_type):
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints(
input=CommandLine.FilenameTypeInfo(arity='+'),
output_dir=CommandLine.DirectoryTypeInfo(ensure_exists=False),
output_name=CommandLine.StringTypeInfo(arity='?'),
build_type=CommandLine.EnumTypeInfo([
"console",
"windows",
],
arity='?'),
manifest_filename=CommandLine.FilenameTypeInfo(arity='?'),
icon_filename=CommandLine.FilenameTypeInfo(arity='?'),
path=CommandLine.DirectoryTypeInfo(arity='*'),
include_module=CommandLine.StringTypeInfo(arity='*'),
exclude_module=CommandLine.StringTypeInfo(arity='*'),
package=CommandLine.StringTypeInfo(arity='*'),
distutil_arg=CommandLine.StringTypeInfo(arity='*'),
comments=CommandLine.StringTypeInfo(arity='?'),
company_name=CommandLine.StringTypeInfo(arity='?'),
file_description=CommandLine.StringTypeInfo(arity='?'),
internal_name=CommandLine.StringTypeInfo(arity='?'),
copyright=CommandLine.StringTypeInfo(arity='?'),
trademark=CommandLine.StringTypeInfo(arity='?'),
name=CommandLine.StringTypeInfo(arity='?'),
version=CommandLine.StringTypeInfo(arity='?'),
output_stream=None,
)
def Compile(
input,
output_dir,
output_name=None,
build_type="console",
include_tcl=False,
no_optimize=False,
no_bundle=False,
manifest_filename=None,
icon_filename=None,
path=None,
include_module=None,
exclude_module=None,
package=None,
distutil_arg=None,
comments=None,
company_name=None,
file_description=None,
internal_name=None,
trademark=None,
copyright=None,
name=None,
version=None,
preserve_temp_dir=False,
output_stream=sys.stdout,
no_verbose=False,
):
"""Creates an executable from one or more python files."""
if build_type == "console":
build_type = DistutilsCompilerImpl.BuildType_Console
elif build_type == "windows":
build_type = DistutilsCompilerImpl.BuildType_Windows
else:
assert False, build_type
return CommandLineCompile(
compiler_type,
input,
output_stream,
verbose=not no_verbose,
# Generate compiler options
output_dir=output_dir,
# This compiler options
preserve_temp_dir=preserve_temp_dir,
build_type=build_type,
include_tcl=include_tcl,
no_optimize=no_optimize,
no_bundle=no_bundle,
manifest_filename=manifest_filename,
icon_filename=icon_filename,
paths=path,
includes=include_module,
excludes=exclude_module,
packages=package,
distutil_args=distutil_arg,
output_name=output_name,
comments=comments,
company_name=company_name,
file_description=file_description,
internal_name=internal_name,
copyright=copyright,
trademark=trademark,
name=name,
version=version,
)
# ----------------------------------------------------------------------
return Compile
_script_dir, _script_name = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------
sys.path.insert(0, os.path.join(_script_dir, "GeneratedCode"))
with CallOnExit(lambda: sys.path.pop(0)):
import HooksImplParser # <Unable to import> pylint: disable = E0401
sys.path.insert(0, os.getenv("DEVELOPMENT_ENVIRONMENT_FUNDAMENTAL"))
with CallOnExit(lambda: sys.path.pop(0)):
from RepositoryBootstrap import Constants
from RepositoryBootstrap.Impl.ActivationData import ActivationData
from RepositoryBootstrap.Impl import Utilities
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints( display_sentinel=CommandLine.StringTypeInfo(),
json_filename=CommandLine.FilenameTypeInfo(),
result_filename=CommandLine.FilenameTypeInfo(ensure_exists=False),
output_stream=None,
)
def Commit( display_sentinel,
json_filename,
result_filename,
first=False,
output_stream=sys.stdout,
):
"""Called prior to committing a change locally"""
return _Impl( display_sentinel,
json_filename,
result_filename,
first,
assert os.path.isfile(
original_environment_filename), original_environment_filename
with open(original_environment_filename) as f:
return json.load(f)
# ----------------------------------------------------------------------
def _Display(content, output_stream, decorate):
if not isinstance(content, six.string_types):
content = json.dumps(content)
if decorate:
output_stream.write(
textwrap.dedent("""\
//--//--//--//--//--//--//--//--//--//--//--//--//--//--//--//
{}
//--//--//--//--//--//--//--//--//--//--//--//--//--//--//--//
""").format(content))
else:
output_stream.write(content)
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
if __name__ == "__main__":
try:
sys.exit(CommandLine.Main())
except KeyboardInterrupt:
pass
if PythonActivationActivity.NormalizeScript(script) == PythonActivationActivity.NormalizeScriptResult_Modified:
this_dm.stream.write(" ** The script '{}' was normalized.\n".format(script))
move_func(script, scripts_dest_dir)
except Exception as ex:
this_dm.result = -1
this_dm.stream.write("ERROR: {}\n".format(StringHelpers.LeftJustify( str(ex),
len("ERROR: "),
)))
return dm.result
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints( lib_name=CommandLine.StringTypeInfo(),
pip_arg=CommandLine.StringTypeInfo(arity='*'),
output_stream=None,
)
def Install( lib_name,
pip_arg=None,
output_stream=sys.stdout,
):
"""
A replacement for pip install. Will ensure that already installed python libraries are not modified in-place,
but rather considered as new libraries for the currently activated repository.
"""
pip_args = pip_arg; del pip_arg
repo_root = os.getenv(RepositoryBootstrapConstants.DE_REPO_ROOT_NAME)
class CodeGenerator(
AtomicInputProcessingMixin,
ConditionalInvocationQueryMixin,
MultipleOutputMixin,
CodeGeneratorBase,
):
# ----------------------------------------------------------------------
# | Types
ContextCode = namedtuple("ContextCode", ["filename", "var_name"])
# ----------------------------------------------------------------------
# | Properties
Name = Interface.DerivedProperty("Jinja2CodeGenerator")
Description = Interface.DerivedProperty(
"Processes a Jinja2 template and produces output")
InputTypeInfo = Interface.DerivedProperty(
CommandLine.FilenameTypeInfo(
validation_expression=".+?\.jinja2(?:\..+)?", ), )
# ----------------------------------------------------------------------
# | Methods
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def _GetOptionalMetadata(cls):
return [("jinja2_context", {}), ("jinja2_context_code", []),
("preserve_dir_structure", False), ("ignore_errors", False),
("debug", False)] + super(
CodeGenerator,
cls,
)._GetOptionalMetadata()
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def _GetRequiredMetadataNames(cls):
return ["output_dir"] + super(CodeGenerator,
cls)._GetRequiredMetadataNames()
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def _CreateContext(cls, metadata, status_stream):
jinja2_context = {}
# Load the custom context defined in code
for context_code in metadata["jinja2_context_code"]:
dirname, basename = os.path.split(context_code)
basename = os.path.splitext(basename)[0]
sys.path.insert(0, dirname)
with CallOnExit(lambda: sys.path.pop(0)):
mod = importlib.import_module(basename)
var = getattr(mod, context_code.var_name)
del mod
if isinstance(var, dict):
for k, v in six.iteritems(var):
jinja2_context[k] = v
else:
jinja2_context[context_code.var_name] = var
del metadata["jinja2_context_code"]
# Load the custom context
for k, v in six.iteritems(metadata["jinja2_context"]):
if len(v) == 1:
jinja2_context[k] = v[0]
else:
jinja2_context[k] = v
metadata["jinja2_context"] = jinja2_context
# Calculate the hashes of the input filenames. We will use this information
# during comparison to determine if an input file has changed. It appears
# that this value isn't used, but it is actually used when comparing the
# context of two different invocations.
# ----------------------------------------------------------------------
def CalculateHash(input_filename):
with open(input_filename, "rb") as f:
return hashlib.sha256(f.read()).digest()
# ----------------------------------------------------------------------
metadata["hashes"] = [
CalculateHash(input_filename)
for input_filename in metadata["inputs"]
]
# Get the output filenames
if not metadata["preserve_dir_structure"]:
# ----------------------------------------------------------------------
def GetBaseDir(input_filename):
return ''
# ----------------------------------------------------------------------
else:
if len(metadata["inputs"]) == 1:
common_prefix = os.path.dirname(metadata["inputs"][0])
else:
common_prefix = FileSystem.GetCommonPath(*metadata["inputs"])
# ----------------------------------------------------------------------
def GetBaseDir(input_filename):
return FileSystem.TrimPath(input_filename, common_prefix)
# ----------------------------------------------------------------------
output_filenames = []
for input_filename in metadata["inputs"]:
output_filenames.append(
os.path.join(
metadata["output_dir"],
GetBaseDir(input_filename),
'.'.join([
part
for part in os.path.basename(input_filename).split(".")
if part != "jinja2"
]),
), )
metadata["output_filenames"] = output_filenames
return super(CodeGenerator,
cls)._CreateContext(metadata, status_stream)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def _InvokeImpl(cls, invoke_reason, context, status_stream, verbose_stream,
verbose):
# ----------------------------------------------------------------------
class RelativeFileSystemLoader(FileSystemLoader):
# ----------------------------------------------------------------------
def __init__(
self,
input_filename,
searchpath=None,
*args,
**kwargs,
):
super(RelativeFileSystemLoader, self).__init__(
searchpath=[os.path.dirname(input_filename)] +
(searchpath or []),
*args,
**kwargs)
# ----------------------------------------------------------------------
def get_source(self, environment, template):
method = super(RelativeFileSystemLoader, self).get_source
try:
return method(environment, template)
except exceptions.TemplateNotFound:
for searchpath in reversed(self.searchpath):
potential_template = os.path.normpath(
os.path.join(searchpath,
template).replace('/', os.path.sep))
if os.path.isfile(potential_template):
dirname, basename = os.path.split(
potential_template)
self.searchpath.append(dirname)
return method(environment, template)
raise
# ----------------------------------------------------------------------
with status_stream.DoneManager(display=False, ) as dm:
for index, (input_filename, output_filename) in enumerate(
zip(
context["inputs"],
context["output_filenames"],
)):
status_stream.write(
"Processing '{}' ({} of {})...".format(
input_filename,
index + 1,
len(context["inputs"]),
), )
with dm.stream.DoneManager(
suppress_exceptions=True, ) as this_dm:
try:
# ----------------------------------------------------------------------
def ReadFileFilter(value):
potential_filename = os.path.join(
os.path.dirname(input_filename), value)
if not os.path.isfile(potential_filename):
return "<< '{}' was not found >>".format(
potential_filename)
with open(potential_filename) as f:
return f.read()
# ----------------------------------------------------------------------
loader = RelativeFileSystemLoader(input_filename)
if context["debug"]:
from jinja2 import meta
env = Environment(loader=loader, )
with open(input_filename) as f:
content = env.parse(f.read())
this_dm.stream.write("Variables:\n{}\n".format(
"\n".join([
" - {}".format(var) for var in
meta.find_undeclared_variables(content)
])))
continue
if context["ignore_errors"]:
undef = Undefined
else:
undef = StrictUndefined
env = Environment(
trim_blocks=True,
lstrip_blocks=True,
loader=loader,
undefined=undef,
)
env.tests["valid_file"] = lambda value: os.path.isfile(
os.path.dirname(input_filename), value)
env.filters[
"doubleslash"] = lambda value: value.replace(
"\\", "\\\\")
# Technically speaking, this isn't required as Jinja's import/include/extend functionality
# superseeds this functionality. However, it remains in the name of backwards compatibility.
env.filters["read_file"] = ReadFileFilter
with open(input_filename) as f:
template = env.from_string(f.read())
try:
content = template.render(
**context["jinja2_context"])
except exceptions.UndefinedError as ex:
this_dm.stream.write("ERROR: {}\n".format(str(ex)))
this_dm.result = -1
continue
with open(output_filename, "w") as f:
f.write(content)
except:
this_dm.result = -1
raise
return dm.result
from CommonEnvironment.StreamDecorator import StreamDecorator
from CommonEnvironment import StringHelpers
# ----------------------------------------------------------------------
_script_fullpath = os.path.abspath(
__file__) if "python" in sys.executable.lower() else sys.executable
_script_dir, _script_name = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------
inflect = inflect_mod.engine()
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints(
root_dir=CommandLine.DirectoryTypeInfo(),
output_stream=None,
)
def EntryPoint(
root_dir,
output_stream=sys.stdout,
):
with StreamDecorator(output_stream).DoneManager(
line_prefix='',
prefix="\nResults: ",
suffix='\n',
) as dm:
repositories = []
dm.stream.write(
"\nSearching for repositories in '{}'...".format(root_dir))
def _RedirectEntryPoint(function_name, entry_point, config):
assert function_name
assert entry_point
assert config
# <Use of exec> pylint: disable = W0122
# <Use of eval> pylint: disable = W0123
required_arguments = []
if config.Configurations:
if function_name != "Clean" or config.ConfigurationRequiredOnClean:
assert "configuration" in entry_point.ConstraintsDecorator.Preconditions, function_name
entry_point.ConstraintsDecorator.Preconditions["configuration"] = CommandLine.EnumTypeInfo(config.Configurations)
required_arguments.append("configuration")
if config.RequiresOutputDir:
required_arguments.append("output_dir")
num_required_args = six.get_function_code(entry_point.Function).co_argcount - len(list(six.get_function_defaults(entry_point.Function) or []))
arguments = six.get_function_code(entry_point.Function).co_varnames[:num_required_args]
if required_arguments and list(arguments[:len(required_arguments) + 1]) != required_arguments:
raise Exception("The entry point '{}' should begin with the arguments '{}' ('{}' were found)".format( function_name,
', '.join(required_arguments),
', '.join(arguments),
))
# Dynamically redefine the function so that it prints information that lets the user know the the
# functionality can't be executed on the current platform/configuration/environment.
if config.RequiredDevelopmentEnvironment and config.RequiredDevelopmentEnvironment.lower() not in [ CurrentShell.Name.lower(),
CurrentShell.CategoryName.lower(),
]:
exec(textwrap.dedent(
"""\
def {name}({args}):
sys.stdout.write("\\nINFO: This can only be run on '{env}'.\\n")
return 0
""").format( name=function_name,
args=', '.join(arguments),
env=config.RequiredDevelopmentEnvironment,
))
entry_point.Function = eval(function_name)
elif config.RequiredDevelopmentConfigurations and not _RedirectEntryPoint_IsSupportedDevelopmentConfiguration(config.RequiredDevelopmentConfigurations):
exec(textwrap.dedent(
"""\
def {name}({args}):
sys.stdout.write("\\nINFO: This can only be run in development environments activated with the configurations {configs}.\\n")
return 0
""").format( name=function_name,
args=', '.join(arguments),
configs=', '.join([ "'{}'".format(configuration) for configuration in config.RequiredDevelopmentConfigurations ]),
))
entry_point.Function = eval(function_name)
elif config.DisableIfDependencyEnvironment:
repo_path = os.getenv(RepositoryBootstrapConstants.DE_REPO_ROOT_NAME)
if FileSystem.GetCommonPath(repo_path, inspect.getfile(entry_point.Function)).rstrip(os.path.sep) != repo_path:
exec(textwrap.dedent(
"""\
def {name}({args}):
sys.stdout.write("\\nINFO: This module is not build when invoked as a dependency.\\n")
return 0
""").format( name=function_name,
args=', '.join(arguments),
))
entry_point.Function = eval(function_name)
inflect = inflect_mod.engine()
# ----------------------------------------------------------------------
BUILD_FILENAME = "Build.py"
BUILD_FILENAME_IGNORE = "{}-ignore".format(BUILD_FILENAME)
BUILD_LOG_TEMPLATE = "Build.{mode}.log"
COMPLETE_CONFIGURATION_NAME = "Complete"
# ----------------------------------------------------------------------
@CommandLine.EntryPoint( mode=CommandLine.EntryPoint.Parameter('Defaults to [ "clean", "build", ]'),
)
@CommandLine.Constraints( root_dir=CommandLine.DirectoryTypeInfo(),
output_dir=CommandLine.DirectoryTypeInfo(ensure_exists=False),
mode=CommandLine.StringTypeInfo(arity='*'),
output_stream=None,
)
def Execute( root_dir,
output_dir,
mode=None,
debug_only=False,
release_only=False,
output_stream=sys.stdout,
verbose=False,
):
"""Recursively calls Build files with the desired mode(s)"""
assert os.path.isdir(root_dir), root_dir
from CommonEnvironment import FileSystem
from CommonEnvironment import Process
from CommonEnvironment.Shell.All import CurrentShell
from CommonEnvironment.StreamDecorator import StreamDecorator
from CommonEnvironment import TaskPool
# ----------------------------------------------------------------------
_script_fullpath = CommonEnvironment.ThisFullpath()
_script_dir, _script_name = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints(
generator=CommandLine.StringTypeInfo(
arity="?",
),
working_dir=CommandLine.DirectoryTypeInfo(
arity="?",
),
cmake_param=CommandLine.StringTypeInfo(
arity="*",
),
output_stream=None,
)
def Generate(
working_dir=os.getcwd(),
generator=(
None
if os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_USE_DEFAULT_CMAKE_GENERATOR")
else "Ninja"
@CommandLine.EntryPoint(
context=CommandLine.EntryPoint.Parameter(
"Jinja2 context passed to the generator."),
context_code=CommandLine.EntryPoint.Parameter(
"Jinja2 context defined in a python file. The argument should be in the form `<python_filename>:<var_name>`."
),
preserve_dir_structure=CommandLine.EntryPoint.Parameter(
"Preserve the input's directory structure when generating output."),
ignore_errors=CommandLine.EntryPoint.Parameter(
"Populate context values not specified with an empty string rather than generating errors."
),
debug=CommandLine.EntryPoint.Parameter(
"Display context rather than generating output."),
)
@CommandLine.Constraints(
output_dir=CommandLine.DirectoryTypeInfo(ensure_exists=False, ),
input=CommandLine.FilenameTypeInfo(
match_any=True,
arity="+",
),
context=CommandLine.DictTypeInfo(
require_exact_match=False,
arity="*",
),
context_code=CommandLine.StringTypeInfo(
validation_expression="^.+:.+$",
arity="*",
),
output_stream=None,
)
def Generate(
def Generate(
working_dir=os.getcwd(),
generator=(
None
if os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_USE_DEFAULT_CMAKE_GENERATOR")
else "Ninja"
),
cmake_param=None,
force=False,
build=False,
test=False,
output_stream=sys.stdout,
verbose=False,
):
"""Generates build files for multiple build configurations"""
cmake_params = cmake_param
del cmake_param
if test and not build:
raise CommandLine.UsageException(
"'/build' must be provided if '/test' is provided",
)
command_line_template = 'cmake {generator} -S "{working_dir}" -B "{{build_dir}}" -DCMAKE_BUILD_TYPE={{configuration}} {params} {root_dir}'.format(
generator='-G "{}"'.format(generator) if generator else "",
working_dir=working_dir,
params=" ".join(cmake_params),
root_dir=os.path.join(*([".."] * 5))
)
# ----------------------------------------------------------------------
def Callback(test_lock, configuration, build_dir, output_stream, on_status_update):
on_status_update("Generating")
_PrintHeader("Generate Output", output_stream)
if os.path.isdir(build_dir):
if not force:
output_stream.write(
"The output dir '{}' already exists and will not be overwritten.\n".format(
build_dir,
),
)
return 1
FileSystem.RemoveTree(build_dir)
FileSystem.MakeDirs(build_dir)
result = Process.Execute(
command_line_template.format(
build_dir=build_dir,
configuration=configuration,
),
output_stream,
)
if result != 0:
return result
# Create a python file that can be used to clean the directory
existing_items = os.listdir(build_dir)
assert existing_items
with open(os.path.join(build_dir, "Clean.py"), "w") as f:
f.write(
textwrap.dedent(
"""\
#!/usr/bin/env python
import os
import sys
import CommonEnvironment
from CommonEnvironment import CommandLine
from CommonEnvironment import FileSystem
from CommonEnvironment.StreamDecorator import StreamDecorator
# ----------------------------------------------------------------------
_script_fullpath = CommonEnvironment.ThisFullpath()
_script_dir, _script_name = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints(
output_stream=None,
)
def EntryPoint(
all=False,
output_stream=sys.stdout,
):
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\\nResults: ",
suffix="\\n",
) as dm:
existing_items = set([{existing_items_list}])
for item in os.listdir(_script_dir):
if item in existing_items or item == _script_name:
continue
fullpath = os.path.join(_script_dir, item)
dm.stream.write("Removing '{{}}'...".format(fullpath))
with dm.stream.DoneManager():
FileSystem.RemoveItem(fullpath)
cmake_dirs = os.path.join(_script_dir, "CMakeFiles")
if all:
dm.stream.write("Removing '{{}}'...".format(cmake_dirs))
with dm.stream.DoneManager():
FileSystem.RemoveTree(cmake_dirs)
else:
dirs_to_delete = []
for fullpath, _ in FileSystem.WalkDirs(
cmake_dirs,
include_dir_names=[lambda name: os.path.splitext(name)[1] == ".dir"],
):
dirs_to_delete.append(fullpath)
for dir_to_delete in dirs_to_delete:
dm.stream.write("Removing '{{}}'...".format(dir_to_delete))
with dm.stream.DoneManager():
FileSystem.RemoveTree(dir_to_delete)
return dm.result
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
if __name__ == "__main__":
try:
sys.exit(CommandLine.Main())
except KeyboardInterrupt:
pass
""",
).format(
existing_items_list=", ".join(
['"{}"'.format(existing_item) for existing_item in existing_items],
),
),
)
if build:
on_status_update("Building")
_PrintHeader("Build Output", output_stream)
result = _BuildImpl(build_dir, output_stream)
if result != 0:
return result
if test:
on_status_update("Testing (Waiting)")
_PrintHeader("Test Output", output_stream)
with test_lock:
on_status_update("Testing")
result = _TestImpl(build_dir, output_stream)
if result != 0:
return result
return 0
# ----------------------------------------------------------------------
return _Impl(working_dir, output_stream, verbose, Callback)
from CommonEnvironmentEx.Package import InitRelativeImports
# ----------------------------------------------------------------------
_script_fullpath = CommonEnvironment.ThisFullpath()
_script_dir, _script_name = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------
with InitRelativeImports():
from .Schema.Parse import ParseFiles
# ----------------------------------------------------------------------
PLUGINS = GeneratorFactory.CreatePluginMap(
"DEVELOPMENT_ENVIRONMENT_SIMPLE_SCHEMA_PLUGINS",
os.path.join(_script_dir, "Plugins"), sys.stdout)
_PluginTypeInfo = CommandLine.EnumTypeInfo(list(six.iterkeys(PLUGINS)))
# ----------------------------------------------------------------------
def _GetOptionalMetadata(*args, **kwargs):
return __GetOptionalMetadata(*args, **kwargs)
def _CreateContext(*args, **kwargs):
return __CreateContext(*args, **kwargs)
def _Invoke(*args, **kwargs):
return __Invoke(*args, **kwargs)
def Html(
bin_dir=None,
profraw_filename="default.profraw",
profdata_filename="default.profdata",
executable=None,
source_dir=None,
output_filename="code_coverage.html",
force=False,
no_sparse=False,
output_stream=sys.stdout,
verbose=False,
):
"""Generates a HTML file based on *.profdata files"""
executables = executable
del executable
source_dirs = source_dir
del source_dir
if bin_dir is None:
bin_dir = os.getcwd()
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
# Generate the profdata file (if necessary)
profdata_filename = os.path.join(bin_dir, profdata_filename)
if force or not os.path.isfile(profdata_filename):
profraw_filename = os.path.join(bin_dir, profraw_filename)
if not os.path.isfile(profraw_filename):
raise CommandLine.UsageException(
"'{}' does not exist.".format(profraw_filename),
)
dm.stream.write("Creating '{}'...".format(profdata_filename))
with dm.stream.DoneManager(
suffix="\n",
) as this_dm:
FileSystem.MakeDirs(os.path.dirname(profdata_filename))
command_line = 'llvm-profdata merge {sparse} -o "{output_filename}" "{input_filename}"'.format(
sparse="" if no_sparse else "-sparse",
output_filename=profdata_filename,
input_filename=profraw_filename,
)
if verbose:
this_dm.stream.write(
textwrap.dedent(
"""\
Command Line:
{}
""",
).format(command_line),
)
this_dm.result = Process.Execute(command_line, this_dm.stream)
if this_dm.result != 0:
return this_dm.result
# Generate the html
output_filename = os.path.join(bin_dir, output_filename)
dm.stream.write("Creating '{}'...".format(output_filename))
with dm.stream.DoneManager(
suffix="\n",
) as this_dm:
if not executables:
this_dm.stream.write("Finding executables...")
with this_dm.stream.DoneManager(
done_suffix=lambda: "{} found".format(
inflect.no("executable", len(executables)),
),
) as find_dm:
if CurrentShell.ExecutableExtension:
executables = list(
FileSystem.WalkFiles(
bin_dir,
include_file_extensions=[
CurrentShell.ExecutableExtension
],
recurse=False,
),
)
else:
for filename in FileSystem.WalkFiles(
bin_dir,
recurse=False,
):
if os.access(filename, os.X_OK):
executables.append(filename)
FileSystem.MakeDirs(os.path.dirname(output_filename))
command_line = 'llvm-cov show {executables} "-instr-profile={profdata}" -use-color --format html {sources} > "{output_filename}"'.format(
executables=" ".join(
['"{}"'.format(executable) for executable in executables],
),
profdata=profdata_filename,
sources=" ".join(
['"{}"'.format(source_dir) for source_dir in source_dirs],
) if source_dirs else "",
output_filename=output_filename,
)
if verbose:
this_dm.stream.write(
textwrap.dedent(
"""\
Command Line:
{}
""",
).format(command_line),
)
this_dm.result = Process.Execute(command_line, this_dm.stream)
if this_dm.result != 0:
return this_dm.result
return dm.result