def _CreateContext(cls, metadata, status_stream): if metadata["plugin_name"] not in plugin_map: raise CommandLine.UsageException( "'{}' is not a valid plugin".format( metadata["plugin_name"])) plugin = plugin_map[metadata["plugin_name"]].Plugin # Ensure that all plugin settings are present and that they # are the expected type. custom_settings = OrderedDict([ (k, v) for k, v in plugin.GenerateCustomSettingsAndDefaults() ]) plugin_settings = metadata["plugin_settings"] for k, v in six.iteritems(plugin_settings): if k not in custom_settings: raise CommandLine.UsageException( "'{}' is not a valid plugin setting".format(k)) desired_type = type(custom_settings[k]) if type(v) != desired_type: assert isinstance(v, (str, UnicodeDecodeError)), (v, type(v)) plugin_settings[k] = StringSerialization.DeserializeItem( CreateFromPythonType(desired_type), v) for k, v in six.iteritems(custom_settings): if k not in plugin_settings: plugin_settings[k] = v metadata["plugin_settings"] = plugin.PreprocessMetadata( plugin_settings) # Invoke custom functionality context = create_context_func(metadata, plugin) context = plugin.PreprocessContext(context) context["output_filenames"] = [ os.path.join(context["output_dir"], filename) for filename in plugin.GenerateOutputFilenames(context) ] context = plugin.PostprocessContext(context) if postprocess_context_func: context = postprocess_context_func(context, plugin) return super(CodeGenerator, cls)._CreateContext(context, status_stream)
def ToRegex(value): try: return re.compile(value) except: raise CommandLine.UsageException( "'{}' is not a valid regular expression".format(value), )
def Create( profile_name, host, username=None, password=None, port=26, from_name=None, from_email=None, ssl=False, output_stream=sys.stdout, ): """Creates a new SmtpMailer profile""" if not from_name and not from_email: raise CommandLine.UsageException( "'from_name' or 'from_email' must be provided") mailer = SmtpMailer( host, username=username, password=password, port=port, from_name=from_name, from_email=from_email, ssl=ssl, ) mailer.Save(profile_name) output_stream.write( "The profile '{}' has been created.\n".format(profile_name))
def Lcov( bin_dir=None, not_llvm=False, output_dir=None, output_filename="lcov.info", type=None, output_stream=sys.stdout, verbose=False, ): """Generates a LCOV file based on *.gcno files""" bin_dirs = bin_dir del bin_dir if not bin_dirs: bin_dirs.append(os.getcwd()) if len(bin_dirs) > 1 and not output_dir: raise CommandLine.UsageException( "An 'output_dir' must be provided when multiple 'bin_dirs' are parsed", ) if len(bin_dirs) == 1 and not output_dir: output_dir = bin_dirs[0] with StreamDecorator(output_stream).DoneManager( line_prefix="", prefix="\nResults: ", suffix="\n", ) as dm: output_filename = os.path.join(output_dir, output_filename) dm.stream.write("Creating '{}'...".format(output_filename)) with dm.stream.DoneManager() as this_dm: FileSystem.MakeDirs(output_dir) command_line = 'grcov {dirs} -o "{output_filename}"{llvm}{type}'.format( dirs=" ".join(['"{}"'.format(dir) for dir in bin_dirs]), output_filename=output_filename, llvm="" if not_llvm else " --llvm", type="" if type is None else " -t {}".format(type), ) if verbose: this_dm.stream.write( textwrap.dedent( """\ Command Line: {} """, ).format(command_line), ) this_dm.result = Process.Execute(command_line, this_dm.stream) if this_dm.result != 0: return this_dm.result return dm.result
def _Impl(working_dir, output_stream, verbose, callback_func): if not os.path.isfile(os.path.join(working_dir, "CMakeLists.txt")): raise CommandLine.UsageException( "The directory '{}' does not contain the file 'CMakeLists.txt'".format( working_dir, ), ) with StreamDecorator(output_stream).DoneManager( line_prefix="", prefix="\nResults: ", suffix="\n", ) as dm: build_dir_prefix = [ "build", CurrentShell.CategoryName, os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_COMPILER_NAME"), os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_ARCHITECTURE"), ] configuration_types = ["Debug", "Release"] # Tests cannot execute in parallel, as they must be invoked from the build # dir (which impacts the global working directory) test_lock = threading.Lock() # ---------------------------------------------------------------------- def Impl(task_index, output_stream, on_status_update): configuration = configuration_types[task_index] build_dir = os.path.join( *([working_dir] + build_dir_prefix + [configuration]) ) return callback_func( test_lock, configuration, build_dir, output_stream, on_status_update, ) # ---------------------------------------------------------------------- dm.result = TaskPool.Execute( [ TaskPool.Task(configuration_type, Impl) for configuration_type in configuration_types ], dm.stream, progress_bar=True, verbose=verbose, ) return dm.result
def CreateCleanMethod(compiler_type): # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( output_dir=CommandLine.DirectoryTypeInfo(), output_stream=None, ) def Clean( output_dir, output_stream=sys.stdout, ): """Cleans previously compiled output.""" return CommandLineClean(output_dir, output_stream) # ---------------------------------------------------------------------- return Clean
def Build( configuration, output_dir, release_build=False, prerelease_build_name=None, no_build_info=False, keep_temp_dir=False, cmake_generator=( None if os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY_CONFIGURATION") == "universal_linux" or os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_USE_DEFAULT_CMAKE_GENERATOR") else "Ninja" ), output_stream=sys.stdout, verbose=False, ): """Builds the Featurizer Shared Library""" if release_build and prerelease_build_name: raise CommandLine.UsageException( "A prerelese build name cannot be provided with the 'release_build' flag", ) with StreamDecorator(output_stream).DoneManager( line_prefix="", prefix="\nResults: ", suffix="\n", ) as dm: FileSystem.RemoveTree(output_dir) FileSystem.MakeDirs(output_dir) temp_directory = CurrentShell.CreateTempDirectory() # ---------------------------------------------------------------------- def CleanupTempDir(): if keep_temp_dir: dm.stream.write( "\nCMake output has been written to '{}'.\n".format(temp_directory), ) return FileSystem.RemoveTree(temp_directory) # ---------------------------------------------------------------------- with CallOnExit(CleanupTempDir): prev_dir = os.getcwd() os.chdir(temp_directory) with CallOnExit(lambda: os.chdir(prev_dir)): if not release_build: if prerelease_build_name is None: # This value should compare as: # "manual" < "pipeline" prerelease_build_name = "manual" if not no_build_info: now = datetime.datetime.now() prerelease_build_name = "{prerelease_build_name}.{year}.{month}.{day}.{hour}.{minute}.{second}.{configuration}".format( year=now.year, month=now.month, day=now.day, hour=now.hour, minute=now.minute, second=now.second, prerelease_build_name=prerelease_build_name, configuration=configuration.lower(), ) activities = [ ( "Generating cmake Files", 'cmake {generator}-DCMAKE_BUILD_TYPE={configuration} {prerelease_build_name} "{this_dir}"'.format( generator='-G "{}" '.format( cmake_generator, ) if cmake_generator else "", temp_dir=temp_directory, configuration=configuration, this_dir=_script_dir, prerelease_build_name="" if not prerelease_build_name else "-DPRODUCT_VERSION_PRERELEASE_INFO={}".format( prerelease_build_name, ), ), ), ("Building", "cmake --build ."), ] if ( os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY_CONFIGURATION") == "universal_linux" ): activities.append( ( "Verifying Universal Linux Binaries", "libcheck libFeaturizers.so", ), ) activities += [ ("Copying Binaries", _CopyBinaries), ("Copying Data", _CopyData), ("Copying Headers", _CopyHeaders), ] for index, (activity, command_line) in enumerate(activities): dm.stream.write( "{} ({} of {})...".format(activity, index + 1, len(activities)), ) with dm.stream.DoneManager( suffix="\n" if verbose else None, ) as this_dm: sink = six.moves.StringIO() output_streams = [sink] if verbose: output_streams.append( StreamDecorator( this_dm.stream, line_prefix="INFO: ", ), ) this_output_stream = StreamDecorator(output_streams) if callable(command_line): this_dm.result = command_line( temp_directory, output_dir, this_output_stream, ) else: this_dm.result = Process.Execute( command_line, this_output_stream, ) if this_dm.result != 0: if not verbose: this_dm.stream.write(sink.getvalue()) return this_dm.result return dm.result
from CommonEnvironment import Process from CommonEnvironment.Shell.All import CurrentShell from CommonEnvironment.StreamDecorator import StreamDecorator # ---------------------------------------------------------------------- _script_fullpath = CommonEnvironment.ThisFullpath() _script_dir, _script_name = os.path.split(_script_fullpath) # ---------------------------------------------------------------------- CONFIGURATIONS = ["Debug", "Release"] JSON_FILENAME = "Microsoft MLFeaturizers.FileAttributes.json" # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( configuration=CommandLine.EnumTypeInfo(CONFIGURATIONS), output_dir=CommandLine.DirectoryTypeInfo( ensure_exists=False, ), prerelease_build_name=CommandLine.StringTypeInfo( arity="?", ), cmake_generator=CommandLine.StringTypeInfo( arity="?", ), output_stream=None, ) def Build( configuration, output_dir, release_build=False,
from CommonEnvironment.CallOnExit import CallOnExit from CommonEnvironment import CommandLine from CommonEnvironment import FileSystem from CommonEnvironment import Process from CommonEnvironment.Shell.All import CurrentShell # ---------------------------------------------------------------------- _script_fullpath = CommonEnvironment.ThisFullpath() _script_dir, _script_name = os.path.split(_script_fullpath) # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( arg=CommandLine.StringTypeInfo( arity="+", ), output_stream=None, ) def EntryPoint( arg, output_stream=sys.stdout, ): args = arg del arg # One of the args will be the filename input_filename = None for arg in args: if arg.startswith("-assume-filename="):
import pandas as pd import CommonEnvironment from CommonEnvironment import CommandLine from CommonEnvironment import FileSystem from CommonEnvironment.StreamDecorator import StreamDecorator # ---------------------------------------------------------------------- _script_fullpath = CommonEnvironment.ThisFullpath() _script_dir, _script_name = os.path.split(_script_fullpath) # ---------------------------------------------------------------------- @CommandLine.EntryPoint() @CommandLine.Constraints( zipped_input_filename=CommandLine.FilenameTypeInfo(), output_stream=None, ) def EntryPoint( zipped_input_filename, output_stream=sys.stdout, ): """Generates JSON files based on data previously pickled""" with StreamDecorator(output_stream).DoneManager( line_prefix="", prefix="\nResults: ", suffix="\n", ) as dm: output_dir = os.path.join(_script_dir, "..", "GeneratedCode")
from CommonEnvironment import CommandLine from CommonEnvironment import FileSystem from CommonEnvironment import Process from CommonEnvironment.Shell.All import CurrentShell from CommonEnvironment.StreamDecorator import StreamDecorator # ---------------------------------------------------------------------- _script_fullpath = CommonEnvironment.ThisFullpath() _script_dir, _script_name = os.path.split(_script_fullpath) # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( output_dir=CommandLine.DirectoryTypeInfo(ensure_exists=False), output_stream=None, ) def Build( output_dir, output_stream=sys.stdout, ): paths = [] includes = [] excludes = [] command_line = '"{script}" Compile "/input={input}" "/output_dir={output_dir}" /no_bundle /no_optimize {paths}{includes}{excludes}' \ .format( script=CurrentShell.CreateScriptName("CxFreezeCompiler"), input=os.path.join(_script_dir, "Backup.py"), output_dir=output_dir, paths='' if not paths else " {}".format(' '.join([ '"/path={}"'.format(path) for path in paths ])),
def _GenerateRebuild(build_func, clean_func, config): assert build_func assert clean_func assert config # ---------------------------------------------------------------------- def Impl(configuration, output_dir, build_func, clean_func): result = clean_func(configuration, output_dir) if result is not None and result != 0: return result if config.RequiresOutputDir and not os.path.isdir(output_dir): os.makedirs(output_dir) result = build_func(configuration, output_dir) if result is not None and result != 0: return result return 0 # ---------------------------------------------------------------------- if config.Configurations: if config.RequiresOutputDir: # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( configuration=CommandLine.EnumTypeInfo(config.Configurations), output_dir=CommandLine.DirectoryTypeInfo(ensure_exists=False), ) def Rebuild(configuration, output_dir): return Impl(configuration, output_dir, build_func, clean_func) # ---------------------------------------------------------------------- else: # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( configuration=CommandLine.EnumTypeInfo(config.Configurations), ) def Rebuild(configuration): return Impl( configuration, None, lambda cfg, output_dir: build_func(cfg), lambda cfg, output_dir: clean_func(cfg), ) # ---------------------------------------------------------------------- else: if config.RequiresOutputDir: # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( output_dir=CommandLine.DirectoryTypeInfo(ensure_exists=False), ) def Rebuild(output_dir): return Impl( None, output_dir, lambda cfg, output_dir: build_func(output_dir), lambda cfg, output_dir: clean_func(output_dir), ) # ---------------------------------------------------------------------- else: # ---------------------------------------------------------------------- @CommandLine.EntryPoint def Rebuild(): return Impl( None, None, lambda cfg, output_dir: build_func(), lambda cfg, output_dir: clean_func(), ) # ---------------------------------------------------------------------- return Rebuild
# ---------------------------------------------------------------------- _script_fullpath = CommonEnvironment.ThisFullpath() _script_dir, _script_name = os.path.split(_script_fullpath) # ---------------------------------------------------------------------- inflect = inflect_mod.engine() # ---------------------------------------------------------------------- DOXYGEN_EXTENSION = ".doxygen" DOXYGEN_EXTENSION_IGNORE = "{}-ignore".format(DOXYGEN_EXTENSION) # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( code_dir_or_doxygen_filename=CommandLine.FilenameTypeInfo( match_any=True, ), output_dir=CommandLine.DirectoryTypeInfo(ensure_exists=False, ), output_stream=None, ) def EntryPoint( code_dir_or_doxygen_filename, output_dir, output_stream=sys.stdout, verbose=False, ): with StreamDecorator(output_stream).DoneManager( line_prefix="", prefix="\nResults: ", suffix="\n", ) as dm: # Get the doxygen files
error_string = plugin_class.IsValidEnvironment() if error_string is not None: sys.stdout.write("INFO: {}\n".format(error_string)) continue yield plugin_class PLUGINS = OrderedDict([(plugin.Name, plugin) for plugin in EnumeratePlugins()]) del EnumeratePlugins if not PLUGINS: sys.stdout.write("\nWARNING: No plugins were found.\n") sys.exit(1) _PluginTypeInfo = CommandLine.EnumTypeInfo(list(six.iterkeys(PLUGINS))) # ---------------------------------------------------------------------- SUPPORTED_TYPES = set([ "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64", "float", "double", "string", "bool",
if context["explicit_passing_score"] else '', )) if passing_score is not None and score < passing_score: result = -1 if result != 0 and not verbose: status_stream.write(sink.getvalue()) return result # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( input=CommandLine.FilenameTypeInfo(match_any=True, arity='+'), passing_score=CommandLine.FloatTypeInfo(min=0.0, max=10.0, arity='?'), output_stream=None, ) def Verify( input, # <Redefinig built-in type> pylint: disable = W0622 passing_score=None, output_stream=sys.stdout, verbose=False, ): """Verifies the given python input""" inputs = input del input return VerifierMod.CommandLineVerify(
def Main( config, original_args=sys.argv, # <Dangerous default value> pylint: disable = W0102 command_line_arg_prefix='/', command_line_keyword_separator='=', command_line_dict_tag_value_separator=':', verbose=False, output_stream=sys.stdout, ): """Method called in a build file's entry point""" assert config assert original_args assert command_line_arg_prefix assert command_line_keyword_separator assert command_line_dict_tag_value_separator assert output_stream # Some build file functions are required, others are not required = { "build" : lambda ep: _RedirectEntryPoint("Build", ep, config), "clean" : lambda ep: _RedirectEntryPoint("Clean", ep, config), } required_names = set(required.keys()) entry_points = CommandLine.EntryPointInformation.FromModule(sys.modules["__main__"]) for entry_point in entry_points: entry_point_name_lower = entry_point.Name.lower() if entry_point_name_lower in required_names: required[entry_point_name_lower](entry_point) required[entry_point_name_lower] = entry_point required_names.remove(entry_point_name_lower) else: for reserved_name in [ "Rebuild", "Metadata", ]: if entry_point_name_lower == reserved_name.lower(): raise Exception("The name '{}' is reserved and will be automatically generated".format(reserved_name)) if required_names: raise Exception("These methods must be defined: {}".format(', '.join(required_names))) entry_points.append(CommandLine.EntryPointInformation.FromFunction(_GenerateRebuild( required["build"], required["clean"], config, ))) # ---------------------------------------------------------------------- @CommandLine.EntryPoint def Metadata(): sys.stdout.write(str(config)) # ---------------------------------------------------------------------- entry_points.append(CommandLine.EntryPointInformation.FromFunction(Metadata)) config = CompleteConfiguration( [ entry_point.Name for entry_point in entry_points ], config, ) script_description_suffix = None if config.Configurations: script_description_suffix = " Where <configuration> can be:\n\n{}\n".format('\n'.join([ " - {}".format(cfg) for cfg in config.Configurations ])) # Execute stack_frame = inspect.stack()[-1] current_dir = os.getcwd() os.chdir(os.path.dirname(os.path.abspath(stack_frame[1]))) with CallOnExit(lambda: os.chdir(current_dir)): # Ensure that an output directory is created prior to invoking build functionality if config.RequiresOutputDir and len(original_args) >= 2 and original_args[1].lower() in [ "build", "rebuild", ]: output_dir = None if config.Configurations: # Command line is: <script> build <config> <output_dir> ... if len(original_args) >= 4: output_dir = original_args[3] else: # Command line is: <script build <output_dir> if len(original_args) >= 3: output_dir = original_args[2] if output_dir: FileSystem.MakeDirs(output_dir) return CommandLine.Executor( args=original_args, command_line_arg_prefix=command_line_arg_prefix, command_line_keyword_separator=command_line_keyword_separator, command_line_dict_tag_value_separator=command_line_dict_tag_value_separator, script_description=inspect.getmodule(stack_frame[0]).__doc__ or '', script_description_suffix=script_description_suffix, entry_points=entry_points, ).Invoke( verbose=verbose, output_stream=output_stream, )
import CommonEnvironment from CommonEnvironment import CommandLine from CommonEnvironmentEx.SmtpMailer import SmtpMailer # ---------------------------------------------------------------------- _script_fullpath = CommonEnvironment.ThisFullpath() _script_dir, _script_name = os.path.split(_script_fullpath) # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( profile_name=CommandLine.StringTypeInfo(), host=CommandLine.StringTypeInfo(), username=CommandLine.StringTypeInfo(arity='?'), password=CommandLine.StringTypeInfo(arity='?'), port=CommandLine.IntTypeInfo(min=1, arity='?'), from_name=CommandLine.StringTypeInfo(arity='?'), from_email=CommandLine.StringTypeInfo(arity='?'), output_stream=None, ) def Create( profile_name, host, username=None, password=None, port=26, from_name=None,
def CreateCompileMethod(compiler_type): # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( input=CommandLine.FilenameTypeInfo(arity='+'), output_dir=CommandLine.DirectoryTypeInfo(ensure_exists=False), output_name=CommandLine.StringTypeInfo(arity='?'), build_type=CommandLine.EnumTypeInfo([ "console", "windows", ], arity='?'), manifest_filename=CommandLine.FilenameTypeInfo(arity='?'), icon_filename=CommandLine.FilenameTypeInfo(arity='?'), path=CommandLine.DirectoryTypeInfo(arity='*'), include_module=CommandLine.StringTypeInfo(arity='*'), exclude_module=CommandLine.StringTypeInfo(arity='*'), package=CommandLine.StringTypeInfo(arity='*'), distutil_arg=CommandLine.StringTypeInfo(arity='*'), comments=CommandLine.StringTypeInfo(arity='?'), company_name=CommandLine.StringTypeInfo(arity='?'), file_description=CommandLine.StringTypeInfo(arity='?'), internal_name=CommandLine.StringTypeInfo(arity='?'), copyright=CommandLine.StringTypeInfo(arity='?'), trademark=CommandLine.StringTypeInfo(arity='?'), name=CommandLine.StringTypeInfo(arity='?'), version=CommandLine.StringTypeInfo(arity='?'), output_stream=None, ) def Compile( input, output_dir, output_name=None, build_type="console", include_tcl=False, no_optimize=False, no_bundle=False, manifest_filename=None, icon_filename=None, path=None, include_module=None, exclude_module=None, package=None, distutil_arg=None, comments=None, company_name=None, file_description=None, internal_name=None, trademark=None, copyright=None, name=None, version=None, preserve_temp_dir=False, output_stream=sys.stdout, no_verbose=False, ): """Creates an executable from one or more python files.""" if build_type == "console": build_type = DistutilsCompilerImpl.BuildType_Console elif build_type == "windows": build_type = DistutilsCompilerImpl.BuildType_Windows else: assert False, build_type return CommandLineCompile( compiler_type, input, output_stream, verbose=not no_verbose, # Generate compiler options output_dir=output_dir, # This compiler options preserve_temp_dir=preserve_temp_dir, build_type=build_type, include_tcl=include_tcl, no_optimize=no_optimize, no_bundle=no_bundle, manifest_filename=manifest_filename, icon_filename=icon_filename, paths=path, includes=include_module, excludes=exclude_module, packages=package, distutil_args=distutil_arg, output_name=output_name, comments=comments, company_name=company_name, file_description=file_description, internal_name=internal_name, copyright=copyright, trademark=trademark, name=name, version=version, ) # ---------------------------------------------------------------------- return Compile
_script_dir, _script_name = os.path.split(_script_fullpath) # ---------------------------------------------------------------------- sys.path.insert(0, os.path.join(_script_dir, "GeneratedCode")) with CallOnExit(lambda: sys.path.pop(0)): import HooksImplParser # <Unable to import> pylint: disable = E0401 sys.path.insert(0, os.getenv("DEVELOPMENT_ENVIRONMENT_FUNDAMENTAL")) with CallOnExit(lambda: sys.path.pop(0)): from RepositoryBootstrap import Constants from RepositoryBootstrap.Impl.ActivationData import ActivationData from RepositoryBootstrap.Impl import Utilities # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( display_sentinel=CommandLine.StringTypeInfo(), json_filename=CommandLine.FilenameTypeInfo(), result_filename=CommandLine.FilenameTypeInfo(ensure_exists=False), output_stream=None, ) def Commit( display_sentinel, json_filename, result_filename, first=False, output_stream=sys.stdout, ): """Called prior to committing a change locally""" return _Impl( display_sentinel, json_filename, result_filename, first,
assert os.path.isfile( original_environment_filename), original_environment_filename with open(original_environment_filename) as f: return json.load(f) # ---------------------------------------------------------------------- def _Display(content, output_stream, decorate): if not isinstance(content, six.string_types): content = json.dumps(content) if decorate: output_stream.write( textwrap.dedent("""\ //--//--//--//--//--//--//--//--//--//--//--//--//--//--//--// {} //--//--//--//--//--//--//--//--//--//--//--//--//--//--//--// """).format(content)) else: output_stream.write(content) # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- if __name__ == "__main__": try: sys.exit(CommandLine.Main()) except KeyboardInterrupt: pass
if PythonActivationActivity.NormalizeScript(script) == PythonActivationActivity.NormalizeScriptResult_Modified: this_dm.stream.write(" ** The script '{}' was normalized.\n".format(script)) move_func(script, scripts_dest_dir) except Exception as ex: this_dm.result = -1 this_dm.stream.write("ERROR: {}\n".format(StringHelpers.LeftJustify( str(ex), len("ERROR: "), ))) return dm.result # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( lib_name=CommandLine.StringTypeInfo(), pip_arg=CommandLine.StringTypeInfo(arity='*'), output_stream=None, ) def Install( lib_name, pip_arg=None, output_stream=sys.stdout, ): """ A replacement for pip install. Will ensure that already installed python libraries are not modified in-place, but rather considered as new libraries for the currently activated repository. """ pip_args = pip_arg; del pip_arg repo_root = os.getenv(RepositoryBootstrapConstants.DE_REPO_ROOT_NAME)
class CodeGenerator( AtomicInputProcessingMixin, ConditionalInvocationQueryMixin, MultipleOutputMixin, CodeGeneratorBase, ): # ---------------------------------------------------------------------- # | Types ContextCode = namedtuple("ContextCode", ["filename", "var_name"]) # ---------------------------------------------------------------------- # | Properties Name = Interface.DerivedProperty("Jinja2CodeGenerator") Description = Interface.DerivedProperty( "Processes a Jinja2 template and produces output") InputTypeInfo = Interface.DerivedProperty( CommandLine.FilenameTypeInfo( validation_expression=".+?\.jinja2(?:\..+)?", ), ) # ---------------------------------------------------------------------- # | Methods # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- @classmethod @Interface.override def _GetOptionalMetadata(cls): return [("jinja2_context", {}), ("jinja2_context_code", []), ("preserve_dir_structure", False), ("ignore_errors", False), ("debug", False)] + super( CodeGenerator, cls, )._GetOptionalMetadata() # ---------------------------------------------------------------------- @classmethod @Interface.override def _GetRequiredMetadataNames(cls): return ["output_dir"] + super(CodeGenerator, cls)._GetRequiredMetadataNames() # ---------------------------------------------------------------------- @classmethod @Interface.override def _CreateContext(cls, metadata, status_stream): jinja2_context = {} # Load the custom context defined in code for context_code in metadata["jinja2_context_code"]: dirname, basename = os.path.split(context_code) basename = os.path.splitext(basename)[0] sys.path.insert(0, dirname) with CallOnExit(lambda: sys.path.pop(0)): mod = importlib.import_module(basename) var = getattr(mod, context_code.var_name) del mod if isinstance(var, dict): for k, v in six.iteritems(var): jinja2_context[k] = v else: jinja2_context[context_code.var_name] = var del metadata["jinja2_context_code"] # Load the custom context for k, v in six.iteritems(metadata["jinja2_context"]): if len(v) == 1: jinja2_context[k] = v[0] else: jinja2_context[k] = v metadata["jinja2_context"] = jinja2_context # Calculate the hashes of the input filenames. We will use this information # during comparison to determine if an input file has changed. It appears # that this value isn't used, but it is actually used when comparing the # context of two different invocations. # ---------------------------------------------------------------------- def CalculateHash(input_filename): with open(input_filename, "rb") as f: return hashlib.sha256(f.read()).digest() # ---------------------------------------------------------------------- metadata["hashes"] = [ CalculateHash(input_filename) for input_filename in metadata["inputs"] ] # Get the output filenames if not metadata["preserve_dir_structure"]: # ---------------------------------------------------------------------- def GetBaseDir(input_filename): return '' # ---------------------------------------------------------------------- else: if len(metadata["inputs"]) == 1: common_prefix = os.path.dirname(metadata["inputs"][0]) else: common_prefix = FileSystem.GetCommonPath(*metadata["inputs"]) # ---------------------------------------------------------------------- def GetBaseDir(input_filename): return FileSystem.TrimPath(input_filename, common_prefix) # ---------------------------------------------------------------------- output_filenames = [] for input_filename in metadata["inputs"]: output_filenames.append( os.path.join( metadata["output_dir"], GetBaseDir(input_filename), '.'.join([ part for part in os.path.basename(input_filename).split(".") if part != "jinja2" ]), ), ) metadata["output_filenames"] = output_filenames return super(CodeGenerator, cls)._CreateContext(metadata, status_stream) # ---------------------------------------------------------------------- @classmethod @Interface.override def _InvokeImpl(cls, invoke_reason, context, status_stream, verbose_stream, verbose): # ---------------------------------------------------------------------- class RelativeFileSystemLoader(FileSystemLoader): # ---------------------------------------------------------------------- def __init__( self, input_filename, searchpath=None, *args, **kwargs, ): super(RelativeFileSystemLoader, self).__init__( searchpath=[os.path.dirname(input_filename)] + (searchpath or []), *args, **kwargs) # ---------------------------------------------------------------------- def get_source(self, environment, template): method = super(RelativeFileSystemLoader, self).get_source try: return method(environment, template) except exceptions.TemplateNotFound: for searchpath in reversed(self.searchpath): potential_template = os.path.normpath( os.path.join(searchpath, template).replace('/', os.path.sep)) if os.path.isfile(potential_template): dirname, basename = os.path.split( potential_template) self.searchpath.append(dirname) return method(environment, template) raise # ---------------------------------------------------------------------- with status_stream.DoneManager(display=False, ) as dm: for index, (input_filename, output_filename) in enumerate( zip( context["inputs"], context["output_filenames"], )): status_stream.write( "Processing '{}' ({} of {})...".format( input_filename, index + 1, len(context["inputs"]), ), ) with dm.stream.DoneManager( suppress_exceptions=True, ) as this_dm: try: # ---------------------------------------------------------------------- def ReadFileFilter(value): potential_filename = os.path.join( os.path.dirname(input_filename), value) if not os.path.isfile(potential_filename): return "<< '{}' was not found >>".format( potential_filename) with open(potential_filename) as f: return f.read() # ---------------------------------------------------------------------- loader = RelativeFileSystemLoader(input_filename) if context["debug"]: from jinja2 import meta env = Environment(loader=loader, ) with open(input_filename) as f: content = env.parse(f.read()) this_dm.stream.write("Variables:\n{}\n".format( "\n".join([ " - {}".format(var) for var in meta.find_undeclared_variables(content) ]))) continue if context["ignore_errors"]: undef = Undefined else: undef = StrictUndefined env = Environment( trim_blocks=True, lstrip_blocks=True, loader=loader, undefined=undef, ) env.tests["valid_file"] = lambda value: os.path.isfile( os.path.dirname(input_filename), value) env.filters[ "doubleslash"] = lambda value: value.replace( "\\", "\\\\") # Technically speaking, this isn't required as Jinja's import/include/extend functionality # superseeds this functionality. However, it remains in the name of backwards compatibility. env.filters["read_file"] = ReadFileFilter with open(input_filename) as f: template = env.from_string(f.read()) try: content = template.render( **context["jinja2_context"]) except exceptions.UndefinedError as ex: this_dm.stream.write("ERROR: {}\n".format(str(ex))) this_dm.result = -1 continue with open(output_filename, "w") as f: f.write(content) except: this_dm.result = -1 raise return dm.result
from CommonEnvironment.StreamDecorator import StreamDecorator from CommonEnvironment import StringHelpers # ---------------------------------------------------------------------- _script_fullpath = os.path.abspath( __file__) if "python" in sys.executable.lower() else sys.executable _script_dir, _script_name = os.path.split(_script_fullpath) # ---------------------------------------------------------------------- inflect = inflect_mod.engine() # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( root_dir=CommandLine.DirectoryTypeInfo(), output_stream=None, ) def EntryPoint( root_dir, output_stream=sys.stdout, ): with StreamDecorator(output_stream).DoneManager( line_prefix='', prefix="\nResults: ", suffix='\n', ) as dm: repositories = [] dm.stream.write( "\nSearching for repositories in '{}'...".format(root_dir))
def _RedirectEntryPoint(function_name, entry_point, config): assert function_name assert entry_point assert config # <Use of exec> pylint: disable = W0122 # <Use of eval> pylint: disable = W0123 required_arguments = [] if config.Configurations: if function_name != "Clean" or config.ConfigurationRequiredOnClean: assert "configuration" in entry_point.ConstraintsDecorator.Preconditions, function_name entry_point.ConstraintsDecorator.Preconditions["configuration"] = CommandLine.EnumTypeInfo(config.Configurations) required_arguments.append("configuration") if config.RequiresOutputDir: required_arguments.append("output_dir") num_required_args = six.get_function_code(entry_point.Function).co_argcount - len(list(six.get_function_defaults(entry_point.Function) or [])) arguments = six.get_function_code(entry_point.Function).co_varnames[:num_required_args] if required_arguments and list(arguments[:len(required_arguments) + 1]) != required_arguments: raise Exception("The entry point '{}' should begin with the arguments '{}' ('{}' were found)".format( function_name, ', '.join(required_arguments), ', '.join(arguments), )) # Dynamically redefine the function so that it prints information that lets the user know the the # functionality can't be executed on the current platform/configuration/environment. if config.RequiredDevelopmentEnvironment and config.RequiredDevelopmentEnvironment.lower() not in [ CurrentShell.Name.lower(), CurrentShell.CategoryName.lower(), ]: exec(textwrap.dedent( """\ def {name}({args}): sys.stdout.write("\\nINFO: This can only be run on '{env}'.\\n") return 0 """).format( name=function_name, args=', '.join(arguments), env=config.RequiredDevelopmentEnvironment, )) entry_point.Function = eval(function_name) elif config.RequiredDevelopmentConfigurations and not _RedirectEntryPoint_IsSupportedDevelopmentConfiguration(config.RequiredDevelopmentConfigurations): exec(textwrap.dedent( """\ def {name}({args}): sys.stdout.write("\\nINFO: This can only be run in development environments activated with the configurations {configs}.\\n") return 0 """).format( name=function_name, args=', '.join(arguments), configs=', '.join([ "'{}'".format(configuration) for configuration in config.RequiredDevelopmentConfigurations ]), )) entry_point.Function = eval(function_name) elif config.DisableIfDependencyEnvironment: repo_path = os.getenv(RepositoryBootstrapConstants.DE_REPO_ROOT_NAME) if FileSystem.GetCommonPath(repo_path, inspect.getfile(entry_point.Function)).rstrip(os.path.sep) != repo_path: exec(textwrap.dedent( """\ def {name}({args}): sys.stdout.write("\\nINFO: This module is not build when invoked as a dependency.\\n") return 0 """).format( name=function_name, args=', '.join(arguments), )) entry_point.Function = eval(function_name)
inflect = inflect_mod.engine() # ---------------------------------------------------------------------- BUILD_FILENAME = "Build.py" BUILD_FILENAME_IGNORE = "{}-ignore".format(BUILD_FILENAME) BUILD_LOG_TEMPLATE = "Build.{mode}.log" COMPLETE_CONFIGURATION_NAME = "Complete" # ---------------------------------------------------------------------- @CommandLine.EntryPoint( mode=CommandLine.EntryPoint.Parameter('Defaults to [ "clean", "build", ]'), ) @CommandLine.Constraints( root_dir=CommandLine.DirectoryTypeInfo(), output_dir=CommandLine.DirectoryTypeInfo(ensure_exists=False), mode=CommandLine.StringTypeInfo(arity='*'), output_stream=None, ) def Execute( root_dir, output_dir, mode=None, debug_only=False, release_only=False, output_stream=sys.stdout, verbose=False, ): """Recursively calls Build files with the desired mode(s)""" assert os.path.isdir(root_dir), root_dir
from CommonEnvironment import FileSystem from CommonEnvironment import Process from CommonEnvironment.Shell.All import CurrentShell from CommonEnvironment.StreamDecorator import StreamDecorator from CommonEnvironment import TaskPool # ---------------------------------------------------------------------- _script_fullpath = CommonEnvironment.ThisFullpath() _script_dir, _script_name = os.path.split(_script_fullpath) # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( generator=CommandLine.StringTypeInfo( arity="?", ), working_dir=CommandLine.DirectoryTypeInfo( arity="?", ), cmake_param=CommandLine.StringTypeInfo( arity="*", ), output_stream=None, ) def Generate( working_dir=os.getcwd(), generator=( None if os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_USE_DEFAULT_CMAKE_GENERATOR") else "Ninja"
@CommandLine.EntryPoint( context=CommandLine.EntryPoint.Parameter( "Jinja2 context passed to the generator."), context_code=CommandLine.EntryPoint.Parameter( "Jinja2 context defined in a python file. The argument should be in the form `<python_filename>:<var_name>`." ), preserve_dir_structure=CommandLine.EntryPoint.Parameter( "Preserve the input's directory structure when generating output."), ignore_errors=CommandLine.EntryPoint.Parameter( "Populate context values not specified with an empty string rather than generating errors." ), debug=CommandLine.EntryPoint.Parameter( "Display context rather than generating output."), ) @CommandLine.Constraints( output_dir=CommandLine.DirectoryTypeInfo(ensure_exists=False, ), input=CommandLine.FilenameTypeInfo( match_any=True, arity="+", ), context=CommandLine.DictTypeInfo( require_exact_match=False, arity="*", ), context_code=CommandLine.StringTypeInfo( validation_expression="^.+:.+$", arity="*", ), output_stream=None, ) def Generate(
def Generate( working_dir=os.getcwd(), generator=( None if os.getenv("DEVELOPMENT_ENVIRONMENT_CPP_USE_DEFAULT_CMAKE_GENERATOR") else "Ninja" ), cmake_param=None, force=False, build=False, test=False, output_stream=sys.stdout, verbose=False, ): """Generates build files for multiple build configurations""" cmake_params = cmake_param del cmake_param if test and not build: raise CommandLine.UsageException( "'/build' must be provided if '/test' is provided", ) command_line_template = 'cmake {generator} -S "{working_dir}" -B "{{build_dir}}" -DCMAKE_BUILD_TYPE={{configuration}} {params} {root_dir}'.format( generator='-G "{}"'.format(generator) if generator else "", working_dir=working_dir, params=" ".join(cmake_params), root_dir=os.path.join(*([".."] * 5)) ) # ---------------------------------------------------------------------- def Callback(test_lock, configuration, build_dir, output_stream, on_status_update): on_status_update("Generating") _PrintHeader("Generate Output", output_stream) if os.path.isdir(build_dir): if not force: output_stream.write( "The output dir '{}' already exists and will not be overwritten.\n".format( build_dir, ), ) return 1 FileSystem.RemoveTree(build_dir) FileSystem.MakeDirs(build_dir) result = Process.Execute( command_line_template.format( build_dir=build_dir, configuration=configuration, ), output_stream, ) if result != 0: return result # Create a python file that can be used to clean the directory existing_items = os.listdir(build_dir) assert existing_items with open(os.path.join(build_dir, "Clean.py"), "w") as f: f.write( textwrap.dedent( """\ #!/usr/bin/env python import os import sys import CommonEnvironment from CommonEnvironment import CommandLine from CommonEnvironment import FileSystem from CommonEnvironment.StreamDecorator import StreamDecorator # ---------------------------------------------------------------------- _script_fullpath = CommonEnvironment.ThisFullpath() _script_dir, _script_name = os.path.split(_script_fullpath) # ---------------------------------------------------------------------- @CommandLine.EntryPoint @CommandLine.Constraints( output_stream=None, ) def EntryPoint( all=False, output_stream=sys.stdout, ): with StreamDecorator(output_stream).DoneManager( line_prefix="", prefix="\\nResults: ", suffix="\\n", ) as dm: existing_items = set([{existing_items_list}]) for item in os.listdir(_script_dir): if item in existing_items or item == _script_name: continue fullpath = os.path.join(_script_dir, item) dm.stream.write("Removing '{{}}'...".format(fullpath)) with dm.stream.DoneManager(): FileSystem.RemoveItem(fullpath) cmake_dirs = os.path.join(_script_dir, "CMakeFiles") if all: dm.stream.write("Removing '{{}}'...".format(cmake_dirs)) with dm.stream.DoneManager(): FileSystem.RemoveTree(cmake_dirs) else: dirs_to_delete = [] for fullpath, _ in FileSystem.WalkDirs( cmake_dirs, include_dir_names=[lambda name: os.path.splitext(name)[1] == ".dir"], ): dirs_to_delete.append(fullpath) for dir_to_delete in dirs_to_delete: dm.stream.write("Removing '{{}}'...".format(dir_to_delete)) with dm.stream.DoneManager(): FileSystem.RemoveTree(dir_to_delete) return dm.result # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- if __name__ == "__main__": try: sys.exit(CommandLine.Main()) except KeyboardInterrupt: pass """, ).format( existing_items_list=", ".join( ['"{}"'.format(existing_item) for existing_item in existing_items], ), ), ) if build: on_status_update("Building") _PrintHeader("Build Output", output_stream) result = _BuildImpl(build_dir, output_stream) if result != 0: return result if test: on_status_update("Testing (Waiting)") _PrintHeader("Test Output", output_stream) with test_lock: on_status_update("Testing") result = _TestImpl(build_dir, output_stream) if result != 0: return result return 0 # ---------------------------------------------------------------------- return _Impl(working_dir, output_stream, verbose, Callback)
from CommonEnvironmentEx.Package import InitRelativeImports # ---------------------------------------------------------------------- _script_fullpath = CommonEnvironment.ThisFullpath() _script_dir, _script_name = os.path.split(_script_fullpath) # ---------------------------------------------------------------------- with InitRelativeImports(): from .Schema.Parse import ParseFiles # ---------------------------------------------------------------------- PLUGINS = GeneratorFactory.CreatePluginMap( "DEVELOPMENT_ENVIRONMENT_SIMPLE_SCHEMA_PLUGINS", os.path.join(_script_dir, "Plugins"), sys.stdout) _PluginTypeInfo = CommandLine.EnumTypeInfo(list(six.iterkeys(PLUGINS))) # ---------------------------------------------------------------------- def _GetOptionalMetadata(*args, **kwargs): return __GetOptionalMetadata(*args, **kwargs) def _CreateContext(*args, **kwargs): return __CreateContext(*args, **kwargs) def _Invoke(*args, **kwargs): return __Invoke(*args, **kwargs)
def Html( bin_dir=None, profraw_filename="default.profraw", profdata_filename="default.profdata", executable=None, source_dir=None, output_filename="code_coverage.html", force=False, no_sparse=False, output_stream=sys.stdout, verbose=False, ): """Generates a HTML file based on *.profdata files""" executables = executable del executable source_dirs = source_dir del source_dir if bin_dir is None: bin_dir = os.getcwd() with StreamDecorator(output_stream).DoneManager( line_prefix="", prefix="\nResults: ", suffix="\n", ) as dm: # Generate the profdata file (if necessary) profdata_filename = os.path.join(bin_dir, profdata_filename) if force or not os.path.isfile(profdata_filename): profraw_filename = os.path.join(bin_dir, profraw_filename) if not os.path.isfile(profraw_filename): raise CommandLine.UsageException( "'{}' does not exist.".format(profraw_filename), ) dm.stream.write("Creating '{}'...".format(profdata_filename)) with dm.stream.DoneManager( suffix="\n", ) as this_dm: FileSystem.MakeDirs(os.path.dirname(profdata_filename)) command_line = 'llvm-profdata merge {sparse} -o "{output_filename}" "{input_filename}"'.format( sparse="" if no_sparse else "-sparse", output_filename=profdata_filename, input_filename=profraw_filename, ) if verbose: this_dm.stream.write( textwrap.dedent( """\ Command Line: {} """, ).format(command_line), ) this_dm.result = Process.Execute(command_line, this_dm.stream) if this_dm.result != 0: return this_dm.result # Generate the html output_filename = os.path.join(bin_dir, output_filename) dm.stream.write("Creating '{}'...".format(output_filename)) with dm.stream.DoneManager( suffix="\n", ) as this_dm: if not executables: this_dm.stream.write("Finding executables...") with this_dm.stream.DoneManager( done_suffix=lambda: "{} found".format( inflect.no("executable", len(executables)), ), ) as find_dm: if CurrentShell.ExecutableExtension: executables = list( FileSystem.WalkFiles( bin_dir, include_file_extensions=[ CurrentShell.ExecutableExtension ], recurse=False, ), ) else: for filename in FileSystem.WalkFiles( bin_dir, recurse=False, ): if os.access(filename, os.X_OK): executables.append(filename) FileSystem.MakeDirs(os.path.dirname(output_filename)) command_line = 'llvm-cov show {executables} "-instr-profile={profdata}" -use-color --format html {sources} > "{output_filename}"'.format( executables=" ".join( ['"{}"'.format(executable) for executable in executables], ), profdata=profdata_filename, sources=" ".join( ['"{}"'.format(source_dir) for source_dir in source_dirs], ) if source_dirs else "", output_filename=output_filename, ) if verbose: this_dm.stream.write( textwrap.dedent( """\ Command Line: {} """, ).format(command_line), ) this_dm.result = Process.Execute(command_line, this_dm.stream) if this_dm.result != 0: return this_dm.result return dm.result