def _CreateContext(cls, metadata, status_stream):
if metadata["plugin_name"] not in plugin_map:
raise CommandLine.UsageException(
"'{}' is not a valid plugin".format(
metadata["plugin_name"]))
plugin = plugin_map[metadata["plugin_name"]].Plugin
# Ensure that all plugin settings are present and that they
# are the expected type.
custom_settings = OrderedDict([
(k, v) for k, v in plugin.GenerateCustomSettingsAndDefaults()
])
plugin_settings = metadata["plugin_settings"]
for k, v in six.iteritems(plugin_settings):
if k not in custom_settings:
raise CommandLine.UsageException(
"'{}' is not a valid plugin setting".format(k))
desired_type = type(custom_settings[k])
if type(v) != desired_type:
assert isinstance(v,
(str, UnicodeDecodeError)), (v, type(v))
plugin_settings[k] = StringSerialization.DeserializeItem(
CreateFromPythonType(desired_type), v)
for k, v in six.iteritems(custom_settings):
if k not in plugin_settings:
plugin_settings[k] = v
metadata["plugin_settings"] = plugin.PreprocessMetadata(
plugin_settings)
# Invoke custom functionality
context = create_context_func(metadata, plugin)
context = plugin.PreprocessContext(context)
context["output_filenames"] = [
os.path.join(context["output_dir"], filename)
for filename in plugin.GenerateOutputFilenames(context)
]
context = plugin.PostprocessContext(context)
if postprocess_context_func:
context = postprocess_context_func(context, plugin)
return super(CodeGenerator,
cls)._CreateContext(context, status_stream)
def Create(
profile_name,
host,
username=None,
password=None,
port=26,
from_name=None,
from_email=None,
ssl=False,
output_stream=sys.stdout,
):
"""Creates a new SmtpMailer profile"""
if not from_name and not from_email:
raise CommandLine.UsageException(
"'from_name' or 'from_email' must be provided")
mailer = SmtpMailer(
host,
username=username,
password=password,
port=port,
from_name=from_name,
from_email=from_email,
ssl=ssl,
)
mailer.Save(profile_name)
output_stream.write(
"The profile '{}' has been created.\n".format(profile_name))
def ToRegex(value):
try:
return re.compile(value)
except:
raise CommandLine.UsageException(
"'{}' is not a valid regular expression".format(value),
)
def Lcov(
bin_dir=None,
not_llvm=False,
output_dir=None,
output_filename="lcov.info",
type=None,
output_stream=sys.stdout,
verbose=False,
):
"""Generates a LCOV file based on *.gcno files"""
bin_dirs = bin_dir
del bin_dir
if not bin_dirs:
bin_dirs.append(os.getcwd())
if len(bin_dirs) > 1 and not output_dir:
raise CommandLine.UsageException(
"An 'output_dir' must be provided when multiple 'bin_dirs' are parsed",
)
if len(bin_dirs) == 1 and not output_dir:
output_dir = bin_dirs[0]
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
output_filename = os.path.join(output_dir, output_filename)
dm.stream.write("Creating '{}'...".format(output_filename))
with dm.stream.DoneManager() as this_dm:
FileSystem.MakeDirs(output_dir)
command_line = 'grcov {dirs} -o "{output_filename}"{llvm}{type}'.format(
dirs=" ".join(['"{}"'.format(dir) for dir in bin_dirs]),
output_filename=output_filename,
llvm="" if not_llvm else " --llvm",
type="" if type is None else " -t {}".format(type),
)
if verbose:
this_dm.stream.write(
textwrap.dedent(
"""\
Command Line:
{}
""",
).format(command_line),
)
this_dm.result = Process.Execute(command_line, this_dm.stream)
if this_dm.result != 0:
return this_dm.result
return dm.result
def _Impl(working_dir, output_stream, verbose, callback_func):
if not os.path.isfile(os.path.join(working_dir, "CMakeLists.txt")):
raise CommandLine.UsageException(
"The directory '{}' does not contain the file 'CMakeLists.txt'".format(
working_dir,
),
)
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
build_dir_prefix = [
"build",
CurrentShell.CategoryName,
os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_COMPILER_NAME"),
os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_ARCHITECTURE"),
]
configuration_types = ["Debug", "Release"]
# Tests cannot execute in parallel, as they must be invoked from the build
# dir (which impacts the global working directory)
test_lock = threading.Lock()
# ----------------------------------------------------------------------
def Impl(task_index, output_stream, on_status_update):
configuration = configuration_types[task_index]
build_dir = os.path.join(
*([working_dir] + build_dir_prefix + [configuration])
)
return callback_func(
test_lock,
configuration,
build_dir,
output_stream,
on_status_update,
)
# ----------------------------------------------------------------------
dm.result = TaskPool.Execute(
[
TaskPool.Task(configuration_type, Impl)
for configuration_type in configuration_types
],
dm.stream,
progress_bar=True,
verbose=verbose,
)
return dm.result
def Build(
configuration,
output_dir,
release_build=False,
prerelease_build_name=None,
no_build_info=False,
keep_temp_dir=False,
cmake_generator=(
None if os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY_CONFIGURATION") == "universal_linux" or os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_USE_DEFAULT_CMAKE_GENERATOR") else "Ninja"
),
output_stream=sys.stdout,
verbose=False,
):
"""Builds the Featurizer Shared Library"""
if release_build and prerelease_build_name:
raise CommandLine.UsageException(
"A prerelese build name cannot be provided with the 'release_build' flag",
)
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
FileSystem.RemoveTree(output_dir)
FileSystem.MakeDirs(output_dir)
temp_directory = CurrentShell.CreateTempDirectory()
# ----------------------------------------------------------------------
def CleanupTempDir():
if keep_temp_dir:
dm.stream.write(
"\nCMake output has been written to '{}'.\n".format(temp_directory),
)
return
FileSystem.RemoveTree(temp_directory)
# ----------------------------------------------------------------------
with CallOnExit(CleanupTempDir):
prev_dir = os.getcwd()
os.chdir(temp_directory)
with CallOnExit(lambda: os.chdir(prev_dir)):
if not release_build:
if prerelease_build_name is None:
# This value should compare as:
# "manual" < "pipeline"
prerelease_build_name = "manual"
if not no_build_info:
now = datetime.datetime.now()
prerelease_build_name = "{prerelease_build_name}.{year}.{month}.{day}.{hour}.{minute}.{second}.{configuration}".format(
year=now.year,
month=now.month,
day=now.day,
hour=now.hour,
minute=now.minute,
second=now.second,
prerelease_build_name=prerelease_build_name,
configuration=configuration.lower(),
)
activities = [
(
"Generating cmake Files",
'cmake {generator}-DCMAKE_BUILD_TYPE={configuration} {prerelease_build_name} "{this_dir}"'.format(
generator='-G "{}" '.format(
cmake_generator,
) if cmake_generator else "",
temp_dir=temp_directory,
configuration=configuration,
this_dir=_script_dir,
prerelease_build_name="" if not prerelease_build_name else "-DPRODUCT_VERSION_PRERELEASE_INFO={}".format(
prerelease_build_name,
),
),
),
("Building", "cmake --build ."),
]
if (
os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY_CONFIGURATION")
== "universal_linux"
):
activities.append(
(
"Verifying Universal Linux Binaries",
"libcheck libFeaturizers.so",
),
)
activities += [
("Copying Binaries", _CopyBinaries),
("Copying Data", _CopyData),
("Copying Headers", _CopyHeaders),
]
for index, (activity, command_line) in enumerate(activities):
dm.stream.write(
"{} ({} of {})...".format(activity, index + 1, len(activities)),
)
with dm.stream.DoneManager(
suffix="\n" if verbose else None,
) as this_dm:
sink = six.moves.StringIO()
output_streams = [sink]
if verbose:
output_streams.append(
StreamDecorator(
this_dm.stream,
line_prefix="INFO: ",
),
)
this_output_stream = StreamDecorator(output_streams)
if callable(command_line):
this_dm.result = command_line(
temp_directory,
output_dir,
this_output_stream,
)
else:
this_dm.result = Process.Execute(
command_line,
this_output_stream,
)
if this_dm.result != 0:
if not verbose:
this_dm.stream.write(sink.getvalue())
return this_dm.result
return dm.result
def Generate(
working_dir=os.getcwd(),
generator=(
None
if os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_USE_DEFAULT_CMAKE_GENERATOR")
else "Ninja"
),
cmake_param=None,
force=False,
build=False,
test=False,
output_stream=sys.stdout,
verbose=False,
):
"""Generates build files for multiple build configurations"""
cmake_params = cmake_param
del cmake_param
if test and not build:
raise CommandLine.UsageException(
"'/build' must be provided if '/test' is provided",
)
command_line_template = 'cmake {generator} -S "{working_dir}" -B "{{build_dir}}" -DCMAKE_BUILD_TYPE={{configuration}} {params} {root_dir}'.format(
generator='-G "{}"'.format(generator) if generator else "",
working_dir=working_dir,
params=" ".join(cmake_params),
root_dir=os.path.join(*([".."] * 5))
)
# ----------------------------------------------------------------------
def Callback(test_lock, configuration, build_dir, output_stream, on_status_update):
on_status_update("Generating")
_PrintHeader("Generate Output", output_stream)
if os.path.isdir(build_dir):
if not force:
output_stream.write(
"The output dir '{}' already exists and will not be overwritten.\n".format(
build_dir,
),
)
return 1
FileSystem.RemoveTree(build_dir)
FileSystem.MakeDirs(build_dir)
result = Process.Execute(
command_line_template.format(
build_dir=build_dir,
configuration=configuration,
),
output_stream,
)
if result != 0:
return result
# Create a python file that can be used to clean the directory
existing_items = os.listdir(build_dir)
assert existing_items
with open(os.path.join(build_dir, "Clean.py"), "w") as f:
f.write(
textwrap.dedent(
"""\
#!/usr/bin/env python
import os
import sys
import CommonEnvironment
from CommonEnvironment import CommandLine
from CommonEnvironment import FileSystem
from CommonEnvironment.StreamDecorator import StreamDecorator
# ----------------------------------------------------------------------
_script_fullpath = CommonEnvironment.ThisFullpath()
_script_dir, _script_name = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints(
output_stream=None,
)
def EntryPoint(
all=False,
output_stream=sys.stdout,
):
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\\nResults: ",
suffix="\\n",
) as dm:
existing_items = set([{existing_items_list}])
for item in os.listdir(_script_dir):
if item in existing_items or item == _script_name:
continue
fullpath = os.path.join(_script_dir, item)
dm.stream.write("Removing '{{}}'...".format(fullpath))
with dm.stream.DoneManager():
FileSystem.RemoveItem(fullpath)
cmake_dirs = os.path.join(_script_dir, "CMakeFiles")
if all:
dm.stream.write("Removing '{{}}'...".format(cmake_dirs))
with dm.stream.DoneManager():
FileSystem.RemoveTree(cmake_dirs)
else:
dirs_to_delete = []
for fullpath, _ in FileSystem.WalkDirs(
cmake_dirs,
include_dir_names=[lambda name: os.path.splitext(name)[1] == ".dir"],
):
dirs_to_delete.append(fullpath)
for dir_to_delete in dirs_to_delete:
dm.stream.write("Removing '{{}}'...".format(dir_to_delete))
with dm.stream.DoneManager():
FileSystem.RemoveTree(dir_to_delete)
return dm.result
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
if __name__ == "__main__":
try:
sys.exit(CommandLine.Main())
except KeyboardInterrupt:
pass
""",
).format(
existing_items_list=", ".join(
['"{}"'.format(existing_item) for existing_item in existing_items],
),
),
)
if build:
on_status_update("Building")
_PrintHeader("Build Output", output_stream)
result = _BuildImpl(build_dir, output_stream)
if result != 0:
return result
if test:
on_status_update("Testing (Waiting)")
_PrintHeader("Test Output", output_stream)
with test_lock:
on_status_update("Testing")
result = _TestImpl(build_dir, output_stream)
if result != 0:
return result
return 0
# ----------------------------------------------------------------------
return _Impl(working_dir, output_stream, verbose, Callback)
def Html(
bin_dir=None,
profraw_filename="default.profraw",
profdata_filename="default.profdata",
executable=None,
source_dir=None,
output_filename="code_coverage.html",
force=False,
no_sparse=False,
output_stream=sys.stdout,
verbose=False,
):
"""Generates a HTML file based on *.profdata files"""
executables = executable
del executable
source_dirs = source_dir
del source_dir
if bin_dir is None:
bin_dir = os.getcwd()
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
# Generate the profdata file (if necessary)
profdata_filename = os.path.join(bin_dir, profdata_filename)
if force or not os.path.isfile(profdata_filename):
profraw_filename = os.path.join(bin_dir, profraw_filename)
if not os.path.isfile(profraw_filename):
raise CommandLine.UsageException(
"'{}' does not exist.".format(profraw_filename),
)
dm.stream.write("Creating '{}'...".format(profdata_filename))
with dm.stream.DoneManager(
suffix="\n",
) as this_dm:
FileSystem.MakeDirs(os.path.dirname(profdata_filename))
command_line = 'llvm-profdata merge {sparse} -o "{output_filename}" "{input_filename}"'.format(
sparse="" if no_sparse else "-sparse",
output_filename=profdata_filename,
input_filename=profraw_filename,
)
if verbose:
this_dm.stream.write(
textwrap.dedent(
"""\
Command Line:
{}
""",
).format(command_line),
)
this_dm.result = Process.Execute(command_line, this_dm.stream)
if this_dm.result != 0:
return this_dm.result
# Generate the html
output_filename = os.path.join(bin_dir, output_filename)
dm.stream.write("Creating '{}'...".format(output_filename))
with dm.stream.DoneManager(
suffix="\n",
) as this_dm:
if not executables:
this_dm.stream.write("Finding executables...")
with this_dm.stream.DoneManager(
done_suffix=lambda: "{} found".format(
inflect.no("executable", len(executables)),
),
) as find_dm:
if CurrentShell.ExecutableExtension:
executables = list(
FileSystem.WalkFiles(
bin_dir,
include_file_extensions=[
CurrentShell.ExecutableExtension
],
recurse=False,
),
)
else:
for filename in FileSystem.WalkFiles(
bin_dir,
recurse=False,
):
if os.access(filename, os.X_OK):
executables.append(filename)
FileSystem.MakeDirs(os.path.dirname(output_filename))
command_line = 'llvm-cov show {executables} "-instr-profile={profdata}" -use-color --format html {sources} > "{output_filename}"'.format(
executables=" ".join(
['"{}"'.format(executable) for executable in executables],
),
profdata=profdata_filename,
sources=" ".join(
['"{}"'.format(source_dir) for source_dir in source_dirs],
) if source_dirs else "",
output_filename=output_filename,
)
if verbose:
this_dm.stream.write(
textwrap.dedent(
"""\
Command Line:
{}
""",
).format(command_line),
)
this_dm.result = Process.Execute(command_line, this_dm.stream)
if this_dm.result != 0:
return this_dm.result
return dm.result
def Clean(
image_name,
tag=None,
all=False,
output_stream=sys.stdout,
):
"""Cleans previously built content"""
tags = tag
del tag
if all and tags:
raise CommandLine.UsageException(
"Individual tag values should not be provided when 'all' is provided on the command line",
)
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
prev_dir = os.getcwd()
os.chdir(os.path.join(_script_dir, image_name))
with CallOnExit(lambda: os.chdir(prev_dir)):
image_name = "{}/{}".format(DOCKER_USER_NAME, image_name)
if all:
image_id = _GetImageId(image_name, dm)
command_line = "docker image rm --force {}".format(image_id)
else:
command_line = "docker image rm {}".format(image_name)
dm.stream.write("Removing docker image(s)...")
with dm.stream.DoneManager() as this_dm:
this_dm.result, output = Process.Execute(command_line)
if this_dm.result != 0:
# If the user provided tags, it may be that they have already deleted this
# image, in which case this is not an error.
this_dm.stream.write(output)
if tags:
this_dm.result = 0
else:
return this_dm.result
if tags:
dm.stream.write("Removing tags...")
with dm.stream.DoneManager() as tag_dm:
for index, tag in enumerate(tags):
tag_dm.stream.write(
"'{}' ({} of {})...".format(
tag, index + 1, len(tags)), )
with tag_dm.stream.DoneManager() as this_dm:
this_dm.result, output = Process.Execute(
"docker image rm {}:{}".format(
image_name, tag), )
if this_dm.result != 0:
this_dm.stream.write(output)
return this_dm.result
return dm.result