def _CreateContext(cls, metadata, status_stream): if metadata["plugin_name"] not in plugin_map: raise CommandLine.UsageException( "'{}' is not a valid plugin".format( metadata["plugin_name"])) plugin = plugin_map[metadata["plugin_name"]].Plugin # Ensure that all plugin settings are present and that they # are the expected type. custom_settings = OrderedDict([ (k, v) for k, v in plugin.GenerateCustomSettingsAndDefaults() ]) plugin_settings = metadata["plugin_settings"] for k, v in six.iteritems(plugin_settings): if k not in custom_settings: raise CommandLine.UsageException( "'{}' is not a valid plugin setting".format(k)) desired_type = type(custom_settings[k]) if type(v) != desired_type: assert isinstance(v, (str, UnicodeDecodeError)), (v, type(v)) plugin_settings[k] = StringSerialization.DeserializeItem( CreateFromPythonType(desired_type), v) for k, v in six.iteritems(custom_settings): if k not in plugin_settings: plugin_settings[k] = v metadata["plugin_settings"] = plugin.PreprocessMetadata( plugin_settings) # Invoke custom functionality context = create_context_func(metadata, plugin) context = plugin.PreprocessContext(context) context["output_filenames"] = [ os.path.join(context["output_dir"], filename) for filename in plugin.GenerateOutputFilenames(context) ] context = plugin.PostprocessContext(context) if postprocess_context_func: context = postprocess_context_func(context, plugin) return super(CodeGenerator, cls)._CreateContext(context, status_stream)
def Create( profile_name, host, username=None, password=None, port=26, from_name=None, from_email=None, ssl=False, output_stream=sys.stdout, ): """Creates a new SmtpMailer profile""" if not from_name and not from_email: raise CommandLine.UsageException( "'from_name' or 'from_email' must be provided") mailer = SmtpMailer( host, username=username, password=password, port=port, from_name=from_name, from_email=from_email, ssl=ssl, ) mailer.Save(profile_name) output_stream.write( "The profile '{}' has been created.\n".format(profile_name))
def ToRegex(value): try: return re.compile(value) except: raise CommandLine.UsageException( "'{}' is not a valid regular expression".format(value), )
def Lcov( bin_dir=None, not_llvm=False, output_dir=None, output_filename="lcov.info", type=None, output_stream=sys.stdout, verbose=False, ): """Generates a LCOV file based on *.gcno files""" bin_dirs = bin_dir del bin_dir if not bin_dirs: bin_dirs.append(os.getcwd()) if len(bin_dirs) > 1 and not output_dir: raise CommandLine.UsageException( "An 'output_dir' must be provided when multiple 'bin_dirs' are parsed", ) if len(bin_dirs) == 1 and not output_dir: output_dir = bin_dirs[0] with StreamDecorator(output_stream).DoneManager( line_prefix="", prefix="\nResults: ", suffix="\n", ) as dm: output_filename = os.path.join(output_dir, output_filename) dm.stream.write("Creating '{}'...".format(output_filename)) with dm.stream.DoneManager() as this_dm: FileSystem.MakeDirs(output_dir) command_line = 'grcov {dirs} -o "{output_filename}"{llvm}{type}'.format( dirs=" ".join(['"{}"'.format(dir) for dir in bin_dirs]), output_filename=output_filename, llvm="" if not_llvm else " --llvm", type="" if type is None else " -t {}".format(type), ) if verbose: this_dm.stream.write( textwrap.dedent( """\ Command Line: {} """, ).format(command_line), ) this_dm.result = Process.Execute(command_line, this_dm.stream) if this_dm.result != 0: return this_dm.result return dm.result
def _Impl(working_dir, output_stream, verbose, callback_func): if not os.path.isfile(os.path.join(working_dir, "CMakeLists.txt")): raise CommandLine.UsageException( "The directory '{}' does not contain the file 'CMakeLists.txt'".format( working_dir, ), ) with StreamDecorator(output_stream).DoneManager( line_prefix="", prefix="\nResults: ", suffix="\n", ) as dm: build_dir_prefix = [ "build", CurrentShell.CategoryName, os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_COMPILER_NAME"), os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_ARCHITECTURE"), ] configuration_types = ["Debug", "Release"] # Tests cannot execute in parallel, as they must be invoked from the build # dir (which impacts the global working directory) test_lock = threading.Lock() # ---------------------------------------------------------------------- def Impl(task_index, output_stream, on_status_update): configuration = configuration_types[task_index] build_dir = os.path.join( *([working_dir] + build_dir_prefix + [configuration]) ) return callback_func( test_lock, configuration, build_dir, output_stream, on_status_update, ) # ---------------------------------------------------------------------- dm.result = TaskPool.Execute( [ TaskPool.Task(configuration_type, Impl) for configuration_type in configuration_types ], dm.stream, progress_bar=True, verbose=verbose, ) return dm.result
def Build( configuration, output_dir, release_build=False, prerelease_build_name=None, no_build_info=False, keep_temp_dir=False, cmake_generator=( None if os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY_CONFIGURATION") == "universal_linux" or os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_USE_DEFAULT_CMAKE_GENERATOR") else "Ninja" ), output_stream=sys.stdout, verbose=False, ): """Builds the Featurizer Shared Library""" if release_build and prerelease_build_name: raise CommandLine.UsageException( "A prerelese build name cannot be provided with the 'release_build' flag", ) with StreamDecorator(output_stream).DoneManager( line_prefix="", prefix="\nResults: ", suffix="\n", ) as dm: FileSystem.RemoveTree(output_dir) FileSystem.MakeDirs(output_dir) temp_directory = CurrentShell.CreateTempDirectory() # ---------------------------------------------------------------------- def CleanupTempDir(): if keep_temp_dir: dm.stream.write( "\nCMake output has been written to '{}'.\n".format(temp_directory), ) return FileSystem.RemoveTree(temp_directory) # ---------------------------------------------------------------------- with CallOnExit(CleanupTempDir): prev_dir = os.getcwd() os.chdir(temp_directory) with CallOnExit(lambda: os.chdir(prev_dir)): if not release_build: if prerelease_build_name is None: # This value should compare as: # "manual" < "pipeline" prerelease_build_name = "manual" if not no_build_info: now = datetime.datetime.now() prerelease_build_name = "{prerelease_build_name}.{year}.{month}.{day}.{hour}.{minute}.{second}.{configuration}".format( year=now.year, month=now.month, day=now.day, hour=now.hour, minute=now.minute, second=now.second, prerelease_build_name=prerelease_build_name, configuration=configuration.lower(), ) activities = [ ( "Generating cmake Files", 'cmake {generator}-DCMAKE_BUILD_TYPE={configuration} {prerelease_build_name} "{this_dir}"'.format( generator='-G "{}" '.format( cmake_generator, ) if cmake_generator else "", temp_dir=temp_directory, configuration=configuration, this_dir=_script_dir, prerelease_build_name="" if not prerelease_build_name else "-DPRODUCT_VERSION_PRERELEASE_INFO={}".format( prerelease_build_name, ), ), ), ("Building", "cmake --build ."), ] if ( os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY_CONFIGURATION") == "universal_linux" ): activities.append( ( "Verifying Universal Linux Binaries", "libcheck libFeaturizers.so", ), ) activities += [ ("Copying Binaries", _CopyBinaries), ("Copying Data", _CopyData), ("Copying Headers", _CopyHeaders), ] for index, (activity, command_line) in enumerate(activities): dm.stream.write( "{} ({} of {})...".format(activity, index + 1, len(activities)), ) with dm.stream.DoneManager( suffix="\n" if verbose else None, ) as this_dm: sink = six.moves.StringIO() output_streams = [sink] if verbose: output_streams.append( StreamDecorator( this_dm.stream, line_prefix="INFO: ", ), ) this_output_stream = StreamDecorator(output_streams) if callable(command_line): this_dm.result = command_line( temp_directory, output_dir, this_output_stream, ) else: this_dm.result = Process.Execute( command_line, this_output_stream, ) if this_dm.result != 0: if not verbose: this_dm.stream.write(sink.getvalue()) return this_dm.result return dm.result
def Generate( working_dir=os.getcwd(), generator=( None if os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_USE_DEFAULT_CMAKE_GENERATOR") else "Ninja" ), cmake_param=None, force=False, build=False, test=False, output_stream=sys.stdout, verbose=False, ): """Generates build files for multiple build configurations""" cmake_params = cmake_param del cmake_param if test and not build: raise CommandLine.UsageException( "'/build' must be provided if '/test' is provided", ) command_line_template = 'cmake {generator} -S "{working_dir}" -B "{{build_dir}}" -DCMAKE_BUILD_TYPE={{configuration}} {params} {root_dir}'.format( generator='-G "{}"'.format(generator) if generator else "", working_dir=working_dir, params=" ".join(cmake_params), root_dir=os.path.join(*([".."] * 5)) ) # ---------------------------------------------------------------------- def Callback(test_lock, configuration, build_dir, output_stream, on_status_update): on_status_update("Generating") _PrintHeader("Generate Output", output_stream) if os.path.isdir(build_dir): if not force: output_stream.write( "The output dir '{}' already exists and will not be overwritten.\n".format( build_dir, ), ) return 1 FileSystem.RemoveTree(build_dir) FileSystem.MakeDirs(build_dir) result = Process.Execute( command_line_template.format( build_dir=build_dir, configuration=configuration, ), output_stream, ) if result != 0: return result # Create a python file that can be used to clean the directory existing_items = os.listdir(build_dir) assert existing_items with open(os.path.join(build_dir, "Clean.py"), "w") as f: f.write( textwrap.dedent( """\ #!/usr/bin/env python import os import sys import CommonEnvironment from CommonEnvironment import CommandLine from CommonEnvironment import FileSystem from CommonEnvironment.StreamDecorator import StreamDecorator # ---------------------------------------------------------------------- _script_fullpath = CommonEnvironment.ThisFullpath() _script_dir, _script_name = os.path.split(_script_fullpath) # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( output_stream=None, ) def EntryPoint( all=False, output_stream=sys.stdout, ): with StreamDecorator(output_stream).DoneManager( line_prefix="", prefix="\\nResults: ", suffix="\\n", ) as dm: existing_items = set([{existing_items_list}]) for item in os.listdir(_script_dir): if item in existing_items or item == _script_name: continue fullpath = os.path.join(_script_dir, item) dm.stream.write("Removing '{{}}'...".format(fullpath)) with dm.stream.DoneManager(): FileSystem.RemoveItem(fullpath) cmake_dirs = os.path.join(_script_dir, "CMakeFiles") if all: dm.stream.write("Removing '{{}}'...".format(cmake_dirs)) with dm.stream.DoneManager(): FileSystem.RemoveTree(cmake_dirs) else: dirs_to_delete = [] for fullpath, _ in FileSystem.WalkDirs( cmake_dirs, include_dir_names=[lambda name: os.path.splitext(name)[1] == ".dir"], ): dirs_to_delete.append(fullpath) for dir_to_delete in dirs_to_delete: dm.stream.write("Removing '{{}}'...".format(dir_to_delete)) with dm.stream.DoneManager(): FileSystem.RemoveTree(dir_to_delete) return dm.result # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- if __name__ == "__main__": try: sys.exit(CommandLine.Main()) except KeyboardInterrupt: pass """, ).format( existing_items_list=", ".join( ['"{}"'.format(existing_item) for existing_item in existing_items], ), ), ) if build: on_status_update("Building") _PrintHeader("Build Output", output_stream) result = _BuildImpl(build_dir, output_stream) if result != 0: return result if test: on_status_update("Testing (Waiting)") _PrintHeader("Test Output", output_stream) with test_lock: on_status_update("Testing") result = _TestImpl(build_dir, output_stream) if result != 0: return result return 0 # ---------------------------------------------------------------------- return _Impl(working_dir, output_stream, verbose, Callback)
def Html( bin_dir=None, profraw_filename="default.profraw", profdata_filename="default.profdata", executable=None, source_dir=None, output_filename="code_coverage.html", force=False, no_sparse=False, output_stream=sys.stdout, verbose=False, ): """Generates a HTML file based on *.profdata files""" executables = executable del executable source_dirs = source_dir del source_dir if bin_dir is None: bin_dir = os.getcwd() with StreamDecorator(output_stream).DoneManager( line_prefix="", prefix="\nResults: ", suffix="\n", ) as dm: # Generate the profdata file (if necessary) profdata_filename = os.path.join(bin_dir, profdata_filename) if force or not os.path.isfile(profdata_filename): profraw_filename = os.path.join(bin_dir, profraw_filename) if not os.path.isfile(profraw_filename): raise CommandLine.UsageException( "'{}' does not exist.".format(profraw_filename), ) dm.stream.write("Creating '{}'...".format(profdata_filename)) with dm.stream.DoneManager( suffix="\n", ) as this_dm: FileSystem.MakeDirs(os.path.dirname(profdata_filename)) command_line = 'llvm-profdata merge {sparse} -o "{output_filename}" "{input_filename}"'.format( sparse="" if no_sparse else "-sparse", output_filename=profdata_filename, input_filename=profraw_filename, ) if verbose: this_dm.stream.write( textwrap.dedent( """\ Command Line: {} """, ).format(command_line), ) this_dm.result = Process.Execute(command_line, this_dm.stream) if this_dm.result != 0: return this_dm.result # Generate the html output_filename = os.path.join(bin_dir, output_filename) dm.stream.write("Creating '{}'...".format(output_filename)) with dm.stream.DoneManager( suffix="\n", ) as this_dm: if not executables: this_dm.stream.write("Finding executables...") with this_dm.stream.DoneManager( done_suffix=lambda: "{} found".format( inflect.no("executable", len(executables)), ), ) as find_dm: if CurrentShell.ExecutableExtension: executables = list( FileSystem.WalkFiles( bin_dir, include_file_extensions=[ CurrentShell.ExecutableExtension ], recurse=False, ), ) else: for filename in FileSystem.WalkFiles( bin_dir, recurse=False, ): if os.access(filename, os.X_OK): executables.append(filename) FileSystem.MakeDirs(os.path.dirname(output_filename)) command_line = 'llvm-cov show {executables} "-instr-profile={profdata}" -use-color --format html {sources} > "{output_filename}"'.format( executables=" ".join( ['"{}"'.format(executable) for executable in executables], ), profdata=profdata_filename, sources=" ".join( ['"{}"'.format(source_dir) for source_dir in source_dirs], ) if source_dirs else "", output_filename=output_filename, ) if verbose: this_dm.stream.write( textwrap.dedent( """\ Command Line: {} """, ).format(command_line), ) this_dm.result = Process.Execute(command_line, this_dm.stream) if this_dm.result != 0: return this_dm.result return dm.result
def Clean( image_name, tag=None, all=False, output_stream=sys.stdout, ): """Cleans previously built content""" tags = tag del tag if all and tags: raise CommandLine.UsageException( "Individual tag values should not be provided when 'all' is provided on the command line", ) with StreamDecorator(output_stream).DoneManager( line_prefix="", prefix="\nResults: ", suffix="\n", ) as dm: prev_dir = os.getcwd() os.chdir(os.path.join(_script_dir, image_name)) with CallOnExit(lambda: os.chdir(prev_dir)): image_name = "{}/{}".format(DOCKER_USER_NAME, image_name) if all: image_id = _GetImageId(image_name, dm) command_line = "docker image rm --force {}".format(image_id) else: command_line = "docker image rm {}".format(image_name) dm.stream.write("Removing docker image(s)...") with dm.stream.DoneManager() as this_dm: this_dm.result, output = Process.Execute(command_line) if this_dm.result != 0: # If the user provided tags, it may be that they have already deleted this # image, in which case this is not an error. this_dm.stream.write(output) if tags: this_dm.result = 0 else: return this_dm.result if tags: dm.stream.write("Removing tags...") with dm.stream.DoneManager() as tag_dm: for index, tag in enumerate(tags): tag_dm.stream.write( "'{}' ({} of {})...".format( tag, index + 1, len(tags)), ) with tag_dm.stream.DoneManager() as this_dm: this_dm.result, output = Process.Execute( "docker image rm {}:{}".format( image_name, tag), ) if this_dm.result != 0: this_dm.stream.write(output) return this_dm.result return dm.result