def FormatInlineFuncDesc(content): initial_whitespace = 37 assert MAX_COLUMN_WIDTH > initial_whitespace content = StringHelpers.Wrap( content, MAX_COLUMN_WIDTH - initial_whitespace) return StringHelpers.LeftJustify(content, initial_whitespace)
def OnFundamental( element, *args, **kwargs): # <Unused argument> pylint: disable = W0613 if not isinstance(element.TypeInfo, EnumTypeInfo): return if not element.TypeInfo.FriendlyValues: return name = element.Name reversed_name = "{}_reversed".format(name) prefix = "{} = OrderedDict".format(name) reversed_prefix = "{} = OrderedDict".format(reversed_name) f.write( textwrap.dedent( """\ {prefix}{assignments} {name}_max_key_length = len(max({name}.keys(), len)) {name}_max_value_length = len(max({name}.values(), len)) {reversed_prefix}{reversed_assignments} {reversed_name}_max_key_length = len(max({reversed_name}.keys(), len)) {reversed_name}_max_value_length = len(max({reversed_name}.values(), len)) """, ).format( prefix=prefix, assignments=StringHelpers.LeftJustify( "([ {}\n ])".format( "\n ".join([ '( "{}", "{}" ),'.format(v, fv) for v, fv in six.moves.zip( element.TypeInfo.Values, element.TypeInfo.FriendlyValues) ]), ), len(prefix), ), name=name, reversed_prefix=reversed_prefix, reversed_assignments=StringHelpers.LeftJustify( "([ {}\n ])".format( "\n ".join([ '( "{}", "{}" ),'.format(fv, v) for v, fv in six.moves.zip( element.TypeInfo.Values, element.TypeInfo.FriendlyValues) ]), ), len(reversed_prefix), ), reversed_name=reversed_name, ), ) nonlocals.wrote_value = True
def OnSimple_VisitedChildren(element): content = fundamental_type_info_visitor.Accept( element.TypeInfo.Items[ element.FundamentalAttributeName]) if content.startswith("<xsd:restriction"): f.write( textwrap.dedent( """\ <xsd:simpleType name="_{}_Item_content"> {} </xsd:simpleType> """, ).format( element.DottedName, StringHelpers.LeftJustify( content.strip(), 2)), ) content = "_{}_Item_content".format( element.DottedName) f.write( textwrap.dedent( """\ <xsd:complexType name="_{name}_Item"> <xsd:simpleContent> <xsd:extension base="{base}"> {attributes} </xsd:extension> </xsd:simpleContent> </xsd:complexType> """, ).format( name=element.DottedName, base=content, attributes=StringHelpers.LeftJustify( "".join([ textwrap.dedent( """\ <xsd:attribute name="{name}" use="{use}" type="{type}" /> """, ).format( name=attribute.Name, use="optional" if attribute.TypeInfo.Arity. IsOptional else "required", type=GetBaseTypeName( attribute.Resolve()), ) for attribute in element.Attributes ], ), 6, ).rstrip(), ), )
def GetDefOutputStatementsConstraintsAndSuffix(self): # ---------------------------------------------------------------------- def ToOrtTypeString(value): return value.replace("std::", "").replace("_t", "") # ---------------------------------------------------------------------- output_statements = [] for index, member in enumerate(self._custom_struct.members): output_statements.append( '.Output({index}, "{name}", "{desc}", "{type}")'.format( index=index, name=member.name, desc=member.description or "No information available", type=self._type_to_template_lookup[member.type], ), ) return ( output_statements, OrderedDict( [ (k, [ToOrtTypeString(v)]) for k, v in six.iteritems(self._template_to_types) ], ), textwrap.dedent( """\ .TypeAndShapeInferenceFunction( [](ONNX_NAMESPACE::InferenceContext& ctx) {{ const bool has_shape = hasInputShape(ctx, 1); {statements} }} ) """, ).format( statements=StringHelpers.LeftJustify( "\n".join( [ textwrap.dedent( """\ propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_{type}, {index}); if(has_shape) {{ propagateShapeFromInputToOutput(ctx, 1, {index}); }} """, ).format( index=index, type=ToOrtTypeString(member.type).upper(), ) for index, member in enumerate(self._custom_struct.members) ], ), 6, ), ) )
def OnFundamental(self, element): python_name = ToPythonName(element) statement = "{type_info}.{method_prefix}Item({python_name}_TypeInfo, {item_statement}, **{serialize_args})".format( type_info=self._type_info_serialization_name, method_prefix=self._method_prefix, python_name=python_name, item_statement=self._source_writer.GetFundamental("item", element), serialize_args=self._custom_serialize_item_args, ) if not element.IsAttribute: statement = self._dest_writer.CreateFundamentalElement( element, statement) self._output_stream.write( textwrap.dedent( """\ # ---------------------------------------------------------------------- @classmethod def _{python_name}_Item(cls, item): return {statement} """, ).format( python_name=python_name, statement=StringHelpers.LeftJustify(statement, 4).strip(), ), )
def Write(name, items, is_os_specific_func): cols = [ 40, 9, 120, ] template = "{name:<%d} {type:<%d} {fullpath:<%d}" % tuple(cols) output_stream.write(textwrap.dedent( """\ {sep} {name} {sep} {header} {underline} {content} """).format( sep='=' * len(name), name=name, header=template.format( name="Name", type="Type", fullpath="Path", ), underline=template.format(**{ k : v for k, v in zip( [ "name", "type", "fullpath", ], [ '-' * col for col in cols ], ) }), content="No items" if not items else StringHelpers.LeftJustify( '\n'.join([ template.format( name="{}{}".format( os.path.basename(item), ' *' if is_os_specific_func(item) else '', ), type="Directory" if os.path.isdir(item) else "File", fullpath=item, ) for item in items ]), 2, ), ))
def GenerateContent(root, is_root): items = [] for k, v in six.iteritems(root): if isinstance(v, six.string_types): items.append( '<repo root="{}" shortname="{}" />\n'.format( v, os.path.basename(k), )) else: tag_name = "allgroup" if is_root else "group" items.append( textwrap.dedent("""\ <{tag_name} name="{name}"> {content} </{tag_name}> """).format( tag_name=tag_name, name=k, content=StringHelpers.LeftJustify( GenerateContent(v, False), 2).rstrip(), )) return ''.join(items)
def Execute(invoke_reason, context, status_stream, verbose_stream, verbose): # <unused argument> pylint: disable = W0613 assert len( context["output_filenames"]) == 1, context["output_filenames"] output_filename = context["output_filenames"][0] status_stream.write("Writing '{}'...".format(output_filename)) with status_stream.DoneManager(): sink = six.StringIO() CommonEnvironment.Describe( context, output_stream=sink, ) sink = sink.getvalue() with open(output_filename, "w") as f: f.write( textwrap.dedent( """\ # ---------------------------------------------------------------------- # | # | Debug Output - Execute # | # ---------------------------------------------------------------------- invoke_reason: {invoke_reason} context: {context} """, ).format( invoke_reason=invoke_reason, context=StringHelpers.LeftJustify(sink.strip(), 4), ), )
def UniqueName(): parts = [] e = element while e: parts.append(StringHelpers.ToPascalCase(e.Name)) e = e.Parent return "_".join(reversed(parts))
def Write(content, prefix, result): ClearTempStatus() message = StringHelpers.LeftJustify( "{}: {}\n".format(prefix, content.strip()), len(prefix), ) if nonlocals.first_write: message = "{}{}".format(dm.stream._line_prefix(0), message) nonlocals.first_write = False nonlocals.dm_write_ref(message) nonlocals.reset_content = False if ( result is not None and (dm.result in [ None, 0, ] or (dm.result > 0 and result < 0)) ): dm.result = result
def OnFundamental(element): content = fundamental_type_info_visitor.Accept( element.TypeInfo) if content.startswith("<xsd:restriction"): f.write( textwrap.dedent( """\ <xsd:simpleType name="_{}_Item"> {} </xsd:simpleType> """, ).format( element.DottedName, StringHelpers.LeftJustify( content, 2).strip()), ) else: element._xsd_base_type = content
def FromReference(cls, referencing_element, referenced_element): """Creates a relationship from two elements""" if getattr(referencing_element, "backref_is_one_to_one", False): relationship_type = cls.RelationshipType.OneToOne elif (referencing_element.TypeInfo.Arity.IsOneOrMore or referencing_element.TypeInfo.Arity.IsZeroOrMore): relationship_type = cls.RelationshipType.ManyToMany else: relationship_type = cls.RelationshipType.OneToMany if getattr(referencing_element, "backref", False): backref_name = getattr( referencing_element, "backref_name", referencing_element.Parent.Object.SingularSnakeName if relationship_type == cls.RelationshipType.OneToOne else referencing_element.Parent.Object.PluralSnakeName, ) is_backref_optional = True is_backref_mutable = referencing_element.mutable and relationship_type == cls.RelationshipType.ManyToMany else: backref_name = None is_backref_optional = None is_backref_mutable = None return cls( referencing_element, relationship_type, referencing_element.Parent.Object, referenced_element.Object, is_optional=referencing_element.TypeInfo.Arity.Min == 0, is_mutable=getattr(referencing_element, "mutable", False), is_parent_child=False, reference_name=StringHelpers.ToSnakeCase( referencing_element.Name if relationship_type == cls.RelationshipType.ManyToMany else referencing_element.Name), backref_name=backref_name, is_backref_optional=is_backref_optional, is_backref_mutable=is_backref_mutable, )
def WriteExtensions(name, items): cols = [ 120, ] template = "{fullpath:<%d}" % tuple(cols) output_stream.write(textwrap.dedent( """\ {sep} {name} {sep} {header} {underline} {content} """).format( sep='=' * len(name), name=name, header=template.format(fullpath="Path"), underline=template.format(**{ k : v for k, v in zip( [ "fullpath", ], [ '-' * col for col in cols ], ) }), content="No items" if not items else StringHelpers.LeftJustify( '\n'.join(items), 2, ), ))
def Usage( self, error=None, error_stream=sys.stderr, verbose=False, potential_method_name=None, ): error_stream.write( textwrap.dedent("""\ {desc}{prefix} Usage: """).format( desc=StringHelpers.Wrap(self.ScriptDescription, MAX_COLUMN_WIDTH), prefix='' if not self.ScriptDescriptionPrefix else "\n\n{}".format(self.ScriptDescriptionPrefix), )) indented_stream = StreamDecorator(error_stream, line_prefix=" ") # Narrow the list down if help was requested for a single method entry_points = self.EntryPoints if len(self.EntryPoints) > 1 and len(self.Args) >= 2: potential_method_name = self.Args[1].lower() for entry_point in entry_points: if entry_point.Name.lower() == potential_method_name: entry_points = [ entry_point, ] break # Display a single item or multiple items if len(entry_points) == 1: standard, verbose_desc = self._GenerateUsageInformation( entry_points[0]) # Add the method name if necessary if len(self.EntryPoints) > 1: if '\n' in standard: standard = "\n {}{}".format( entry_points[0].Name, standard, ) else: standard = "{} {}".format( entry_points[0].Name, standard, ) if verbose: standard = "{}\n\n{}".format(standard, verbose_desc) indented_stream.write(" {} {}\n\n".format( self.ScriptName, StringHelpers.LeftJustify(standard, 4), )) else: # ---------------------------------------------------------------------- def FormatInlineFuncDesc(content): initial_whitespace = 37 assert MAX_COLUMN_WIDTH > initial_whitespace content = StringHelpers.Wrap( content, MAX_COLUMN_WIDTH - initial_whitespace) return StringHelpers.LeftJustify(content, initial_whitespace) # ---------------------------------------------------------------------- indented_stream.write( textwrap.dedent("""\ {script_name} <command> [args] Where '<command>' can be one of the following: ---------------------------------------------- """).format(script_name=self.ScriptName, )) for entry_point in entry_points: indented_stream.write(" - {name:<30} {desc}\n".format( name=entry_point.Name, desc=FormatInlineFuncDesc(entry_point.Description), )) indented_stream.write('\n') for entry_point in entry_points: intro = "When '<command>' is '{}':".format(entry_point.Name) standard, verbose_desc = self._GenerateUsageInformation( entry_point) # Insert the function name as an argument if '\n' in standard: multi_line_args = True standard = " {}{}".format( entry_point.Name, StringHelpers.LeftJustify(standard, 4)) else: multi_line_args = False standard = "{} {}".format(entry_point.Name, standard) if verbose: standard = "{}\n\n{}".format( standard, StringHelpers.LeftJustify(verbose_desc, 4, skip_first_line=False), ) indented_stream.write( textwrap.dedent("""\ {intro} {sep} {script_name}{newline}{standard} """).format( intro=intro, sep='-' * len(intro), script_name=self.ScriptName, newline='\n' if multi_line_args else ' ', standard=standard, )) if self.ScriptDescriptionSuffix: error_stream.write("\n{}\n".format( self.ScriptDescriptionSuffix.strip())) if not verbose: error_stream.write( textwrap.dedent("""\ Run "{script_name} {prefix}?" for additional information. """).format( script_name=self.ScriptName, prefix=self.CommandLineArgPrefix, )) if error: error = "\n\nERROR: {}\n".format( StringHelpers.LeftJustify(error, len("ERROR: "))) try: import colorama colorama.init(autoreset=True) error_stream = sys.stderr error_stream.write("{}{}{}".format( colorama.Fore.RED, colorama.Style.BRIGHT, error, )) except ImportError: error_stream.write(error) return -1
def Move( no_move=False, ignore_warnings=False, output_stream=sys.stdout, ): """Moves any new python libraries to the appropriate Libraries folder associated with the activated repository.""" with StreamDecorator(output_stream).DoneManager( line_prefix='', prefix="\nResults: ", suffix='\n', ) as dm: if no_move: dm.stream.write("***** Output is for information only; nothing will be moved. *****\n\n") move_func = lambda *args, **kwargs: None else: # ---------------------------------------------------------------------- def Impl(source_dir_or_filename, dest_dir): # shutil.move won't overwrite files, so use distutils (which will) if os.path.isdir(source_dir_or_filename): import distutils.dir_util distutils.dir_util.copy_tree(source_dir_or_filename, os.path.join(dest_dir, os.path.basename(source_dir_or_filename))) FileSystem.RemoveTree(source_dir_or_filename) else: FileSystem.MakeDirs(dest_dir) shutil.move(source_dir_or_filename, dest_dir) # ---------------------------------------------------------------------- move_func = Impl dm.stream.write("Calculating new library content...") with dm.stream.DoneManager(): new_content = _NewLibraryContent.Create(_EnvironmentSettings()) # Group libraries and distinfo bundles # ---------------------------------------------------------------------- class PythonLibrary(object): def __init__(self, fullpath): self.Fullpath = fullpath self.metadata_path = None self.version = None self.scripts = [] # ---------------------------------------------------------------------- libraries = OrderedDict() dm.stream.write("Grouping libraries...") with dm.stream.DoneManager( done_suffix=lambda: "{} found".format(inflect.no("library", len(libraries))), suffix='\n', ) as this_dm: for library_path in new_content.Libraries: basename = os.path.basename(library_path) if ( not basename.endswith(".dist-info") and not basename.endswith(".egg-info") ): libraries[basename] = PythonLibrary(library_path) lowercase_map = { k.lower() : k for k in six.iterkeys(libraries) } # Extract library metadata for library_path in new_content.Libraries: if os.path.isfile(library_path): continue basename = os.path.basename(library_path) if not ( basename.endswith(".dist-info") or basename.endswith(".egg-info") ): continue index = basename.find('-') if index == -1: this_dm.result = this_dm.result or 1 this_dm.stream.write("WARNING: The library name for '{}' could not be extracted.\n".format(library_path)) continue potential_name = basename[:index] potential_names = [ potential_name, "{}.py".format(potential_name), ] # Try to find the python library based on the potential names python_library = None for potential_name in potential_names: python_library = libraries.get(potential_name, None) if python_library is None: library_key = lowercase_map.get(potential_name.lower(), None) if library_key is not None: python_library = libraries.get(library_key, None) if python_library is not None: break if python_library is None: this_dm.result = this_dm.result or 1 this_dm.stream.write("WARNING: The library name '{}' was not found ({}).\n".format(potential_names[0], library_path)) continue if basename.endswith(".dist-info"): python_library.metadata_path = library_path version = None if version is None: metadata_filename = os.path.join(library_path, "metadata.json") if os.path.isfile(metadata_filename): with open(metadata_filename) as f: data = json.load(f) if "version" not in data: this_dm.result = -1 this_dm.stream.write("ERROR: 'version' was not found in '{}'.\n".format(library_path)) continue version = data["version"] if version is None: metadata_filename = os.path.join(library_path, "METADATA") if os.path.isfile(metadata_filename): for line in open(metadata_filename).readlines(): if line.startswith("Version:"): version = line[len("Version:"):].strip() break if version is None: this_dm.result = -1 this_dm.stream.write("ERROR: Metadata was not found for '{}'.\n".format(library_path)) continue python_library.version = version elif basename.endswith(".egg-info"): python_library.metadata_path = library_path metadata_filename = os.path.join(library_path, "PKG-INFO") if not os.path.isfile(metadata_filename): this_dm.result = -1 this_dm.stream.write("ERROR: Metadata was not found for '{}'.\n".format(metadata_filename)) continue version = None for line in open(metadata_filename).readlines(): if line.startswith("Version:"): version = line[len("Version:"):].strip() break if version is None: this_dm.result = -1 this_dm.stream.write("ERROR: 'Version:' was not found in '{}'.\n".format(metadata_filename)) continue python_library.version = version else: assert False, basename # Eliminate all library info where we couldn't extract the version for library_name in list(six.iterkeys(libraries)): if libraries[library_name].version is None: this_dm.result = this_dm.result or 1 this_dm.stream.write("WARNING: Version information was not found for '{}'.\n".format(library_name)) libraries.pop(library_name) # Associate scripts with the known library info for script_fullpath in new_content.Scripts: if os.path.isdir(script_fullpath): this_dm.result = this_dm.result or 1 this_dm.stream.write("WARNING: '{}' is a directory and will not be processed.\n".format(script_fullpath)) continue script_name_lower = os.path.splitext(os.path.basename(script_fullpath))[0].lower() found = False for potential_library_name, potential_library_info in six.iteritems(libraries): if potential_library_name.lower() in script_name_lower: potential_library_info.scripts.append(script_fullpath) found = True break if not found: this_dm.result = this_dm.result or 1 this_dm.stream.write("WARNING: The library for the script '{}' could not be found.\n".format(script_fullpath)) if dm.result < 0: return dm.result if not ignore_warnings and dm.result > 0: dm.stream.write("\nWarnings were encountered. To continue execution even with warnings, specify 'ignore_warnings' on the command line.\n") return dm.result if not libraries: return dm.stream.write("Moving content...") with dm.stream.DoneManager( suffix='\n', ) as move_dm: # BugBug: Handle case where existing dest dir isn't os-specific but now needs to be with new changes # ---------------------------------------------------------------------- def DestinationIsOSSpecific(dest_dir): # BugBug: I don't think that this works. Should be looking for os-specific names and then python names. if not os.path.isdir(dest_dir): return False found_one = False for item in os.listdir(dest_dir): if not item.startswith("python"): return False fullpath = os.path.join(dest_dir, item) if not os.path.isdir(fullpath): return False found_one = True return found_one # ---------------------------------------------------------------------- python_version_dir = "python{}".format(os.getenv("DEVELOPMENT_ENVIRONMENT_PYTHON_VERSION").split('.')[0]) display_template = "{0:<50} -> {1}\n" for index, (library_name, library_info) in enumerate(six.iteritems(libraries)): move_dm.stream.write("Processing '{}' ({} of {})...".format( library_name, index + 1, len(libraries), )) with move_dm.stream.DoneManager( suffix='\n', ) as this_dm: try: if library_name.endswith(".py"): library_name = library_name[:-len(".py")] dest_dir = os.path.join(os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY"), RepositoryBootstrapConstants.LIBRARIES_SUBDIR, PythonActivationActivity.Name, library_name, "v{}".format(library_info.version)) has_os_specific_dest_dir = False if new_content.HasOSSpecificLibraryExtensions(library_info.Fullpath) or DestinationIsOSSpecific(dest_dir): dest_dir = os.path.join(dest_dir, CurrentShell.CategoryName, python_version_dir) has_os_specific_dest_dir = True # Copy the library library_dest_dir = dest_dir this_dm.stream.write(display_template.format(os.path.basename(library_info.Fullpath), library_dest_dir)) move_func(library_info.Fullpath, library_dest_dir) # Copy the metadata assert library_info.metadata_path metadata_dest_dir = dest_dir this_dm.stream.write(display_template.format(os.path.basename(library_info.metadata_path), metadata_dest_dir)) move_func(library_info.metadata_path, metadata_dest_dir) # Copy the scripts scripts_dest_dir = os.path.join(dest_dir, SCRIPTS_DIR_NAME) if not has_os_specific_dest_dir and bool(next((script for script in library_info.scripts if os.path.splitext(script)[1] in [ CurrentShell.ScriptExtension, CurrentShell.ExecutableExtension, ]), None)): scripts_dest_dir = os.path.join(scripts_dest_dir, CurrentShell.CategoryName) for script in library_info.scripts: this_dm.stream.write(display_template.format("{} [Script]".format(os.path.basename(script)), scripts_dest_dir)) if not no_move: if PythonActivationActivity.NormalizeScript(script) == PythonActivationActivity.NormalizeScriptResult_Modified: this_dm.stream.write(" ** The script '{}' was normalized.\n".format(script)) move_func(script, scripts_dest_dir) except Exception as ex: this_dm.result = -1 this_dm.stream.write("ERROR: {}\n".format(StringHelpers.LeftJustify( str(ex), len("ERROR: "), ))) return dm.result
def _Impl( display_sentinel, json_filename, result_filename, first, output_stream, method_name, parser, ): output_stream = StreamDecorator( output_stream, line_prefix=display_sentinel, ) with open(json_filename) as f: try: data = parser(f.read(), is_root=True) except Exception as ex: output_stream.write("ERROR: {} ({})\n".format(str(ex), ex.stack)) return -1 output_stream.write("Parsing dependencies...") with output_stream.DoneManager(): dependencies = ActivationData.Load(None, None, None).PrioritizedRepositories has_config_specific = False output_stream.write("Validating...") with output_stream.DoneManager() as dm: for index, repository_info in enumerate(dependencies): dm.stream.write("Processing '{}' ({} of {})...".format( repository_info.Name, index + 1, len(dependencies), )) with dm.stream.DoneManager() as this_dm: with Utilities.CustomMethodManager(os.path.join(repository_info.Root, Constants.HOOK_ENVIRONMENT_CUSTOMIZATION_FILENAME), method_name) as method: if not method: continue args = OrderedDict([ ( "data", data ), ( "output_stream", this_dm.stream ), ]) # Get the method args to see if a configuration is requried func_code = six.get_function_code(method) if "configuration" in func_code.co_varnames[:func_code.co_argcount]: args["configuration"] = repository_info.Configuration has_config_specific = True elif not first: # Don't call a config-agnostic method more than once continue try: this_dm.result = Interface.CreateCulledCallable(method)(args) or 0 except Exception as ex: this_dm.stream.write(StringHelpers.LeftJustify( "ERROR: {}\n".format(str(ex).rstrip()), len("ERROR: "), )) this_dm.result = -1 with open(result_filename, 'w') as f: f.write('-1' if dm.result != 0 else '1' if has_config_specific else '0') return dm.result
def _Invoke(cls, context, status_stream, verbose): """Handles the complexities of compiler invocation, ultimately calling _InvokeImpl.""" assert context status_stream = StreamDecorator(status_stream) invoke_reason = cls._GetInvokeReason( context, StreamDecorator(status_stream if verbose else None)) if invoke_reason is None: status_stream.write("No changes were detected.\n") return 0 input_items = cls.GetInputItems(context) assert input_items status_stream.write( cls._GetStatusText(cls.InvokeVerb, context, input_items)) with status_stream.DoneManager() as dm: if verbose: output_items = cls.GetOutputItems(context) if "display_name" in context or len(input_items) == 1: indentation = 4 else: indentation = 8 verbose_stream = StreamDecorator( dm.stream, prefix=StringHelpers.LeftJustify( textwrap.dedent( # <Wrong hanging indentation> pylint: disable = C0330 """\ ======================================== VERBOSE Output {} -> {} ======================================== """ ).format( '\n'.join(input_items), StringHelpers.LeftJustify( '\n'.join(output_items) if output_items else "[None]", 4), ), 2, skip_first_line=False, ), suffix='\n', line_prefix=' ' * indentation, ) status_stream = verbose_stream else: status_stream = dm.stream verbose_stream = StreamDecorator(None) dm.result = cls._InvokeImpl( invoke_reason, context, status_stream, verbose_stream, verbose, ) or 0 if dm.result >= 0: cls._PersistContext(context) return dm.result
def EntryPoint( plugin, input_filename, output_dir, include=None, exclude=None, output_stream=sys.stdout, ): """Generates content based on a configuration file according to the specified plugin""" plugin = PLUGINS[plugin] # ---------------------------------------------------------------------- def ToRegex(value): try: return re.compile(value) except: raise CommandLine.UsageException( "'{}' is not a valid regular expression".format(value), ) # ---------------------------------------------------------------------- includes = [ToRegex(arg) for arg in include] del include excludes = [ToRegex(arg) for arg in exclude] del exclude with StreamDecorator(output_stream).DoneManager( line_prefix="", prefix="\nResults: ", suffix="\n", ) as dm: dm.stream.write("Reading input data...") with dm.stream.DoneManager() as this_dm: try: data = Serialization.Deserialize(input_filename) except Exception as e: this_dm.stream.write( textwrap.dedent( """\ ERROR: {} {} """, ).format( StringHelpers.LeftJustify(str(e), len("ERROR: ")), str(getattr(e, "stack", None)), ), ) this_dm.result = -1 return this_dm.result nonlocals = CommonEnvironment.Nonlocals(skipped=0, ) dm.stream.write("Preprocessing data...") with dm.stream.DoneManager( done_suffix=lambda: "{} were skipped".format( inflect.no("file", nonlocals.skipped), ), suffix=lambda: "\n" if nonlocals.skipped else None, ) as this_dm: # ---------------------------------------------------------------------- def NormalizeEnum(enum): # Simplify the provided enum structure be creating an ordered dictionary with names and values if hasattr(enum, "integer_values"): if len(enum.integer_values) != len(enum.values): raise Exception( "When integer values are specified for an enum, the number of integers must match the number of enums ('{}', '{}')" .format(enum.values, enum.integer_values)) integer_values = enum.integer_values del enum.integer_values else: integer_values = list( range(enum.starting_index, enum.starting_index + len(enum.values))) del enum.starting_index assert len( enum.values) == len(integer_values), (enum.values, integer_values) enum.values = OrderedDict([ (k, v) for k, v in zip(enum.values, integer_values) ]) return enum # ---------------------------------------------------------------------- # Get the global custom structs global_custom_struct_names = set() global_custom_structs = [] for item in data.custom_structs: if item.name in global_custom_struct_names: raise Exception( "The custom struct '{}' has already been defined". format(item.name)) global_custom_struct_names.add(item.name) global_custom_structs.append(item) # Get the global custom enums global_custom_enum_names = set() global_custom_enums = [] for item in data.custom_enums: if item.name in global_custom_enum_names: raise Exception( "The custom enum '{}' has already been defined".format( item.name)) global_custom_enum_names.add(item.name) global_custom_enums.append(NormalizeEnum(item)) # If there are templates at play, preprocess the content and expand the values new_data = [] for item in data.featurizers: if item.status != "Available": this_dm.stream.write( "The status for '{}' is set to '{}' and will not be processed.\n" .format( item.name, item.status, ), ) nonlocals.skipped += 1 continue if excludes and any( exclude.match(item.name) for exclude in excludes): this_dm.stream.write( "'{}' has been explicitly excluded.\n".format( item.name), ) nonlocals.skipped += 1 continue if includes and not any( include.match(item.name) for include in includes): this_dm.stream.write( "'{}' has not been included.\n".format(item.name), ) nonlocals.skipped += 1 continue for enum in getattr(item, "custom_enums", []): NormalizeEnum(enum) if not hasattr(item, "templates"): assert item.type_mappings for mapping in item.type_mappings: new_item = copy.deepcopy(item) new_item.input_type = mapping.input_type new_item.output_type = mapping.output_type new_data.append([new_item]) continue new_data_items = [] for template in item.templates: regex = re.compile(r"\b{}\b".format(template.name)) for template_type in template.types: new_item = copy.deepcopy(item) new_item.template = template_type # Remove the template mapping and list of templates del new_item.templates del new_item.type_mappings for configuration_param in getattr( new_item, "configuration_params", [], ): configuration_param.type = regex.sub( template_type, configuration_param.type, ) for custom_struct in getattr(new_item, "custom_structs", []): if any(gcs for gcs in global_custom_structs if gcs.name == custom_struct.name): raise Exception( "The custom structure '{}' in '{}' has already been defined as a global custom struct.\n" .format(custom_struct.name, item.name)) for member in custom_struct.members: member.type = regex.sub( template_type, member.type) for custom_enum in getattr(new_item, "custom_enums", []): if any(gce for gce in global_custom_enums if gce.name == custom_enum.name): raise Exception( "The custom enum '{}' in '{}' has already been defined as a global custom enum.\n" .format(custom_enum.name, item.name)) custom_enum.underlying_type = regex.sub( template_type, custom_enum.underlying_type) for mapping in item.type_mappings: # TODO: sub all types (for example: map<K, V> if not regex.search( mapping.input_type) and not regex.search( mapping.output_type): continue new_item.input_type = regex.sub( template_type, mapping.input_type) if new_item.input_type != mapping.input_type: new_item.input_type_template_mapping = OrderedDict( [ (template_type, template.name), ], ) new_item.output_type = regex.sub( template_type, mapping.output_type) if new_item.output_type != mapping.output_type: new_item.output_type_template_mapping = OrderedDict( [ (template_type, template.name), ], ) # This will end up copying one more time than needed, but I couldn't think of a better way for now. new_data_items.append(copy.deepcopy(new_item)) new_data.append(new_data_items) data = new_data # Validate parameters dm.stream.write("Validating types...") with dm.stream.DoneManager(): for items in data: for item in items: # ---------------------------------------------------------------------- def IsSupportedType(typename): for potential_type in SUPPORTED_TYPES: if hasattr(potential_type, "match"): if potential_type.match(typename): return True elif typename == potential_type: return True return False # ---------------------------------------------------------------------- def IsCustomStructType(typename): return any(custom_struct for custom_struct in itertools.chain( getattr(item, "custom_structs", []), global_custom_structs) if custom_struct.name == typename) # ---------------------------------------------------------------------- def IsCustomEnumType(typename): return any(custom_enum for custom_enum in itertools.chain( getattr(item, "custom_enums", []), global_custom_enums) if custom_enum.name == typename) # ---------------------------------------------------------------------- input_type = item.input_type if input_type.endswith("?"): input_type = input_type[:-1] if (not IsSupportedType(input_type) and not IsCustomStructType(input_type) and not IsCustomEnumType(input_type)): raise Exception( "The input type '{}' defined in '{}' is not valid." .format( input_type, item.name, ), ) from None output_type = item.output_type if output_type.endswith("?"): output_type = output_type[:-1] if (not IsSupportedType(output_type) and not IsCustomStructType(output_type) and not IsCustomEnumType(output_type)): raise Exception( "The output type '{}' defined in '{}' is not valid." .format( output_type, item.name, ), ) from None dm.stream.write("Generating content...") with dm.stream.DoneManager() as this_dm: FileSystem.MakeDirs(output_dir) # ---------------------------------------------------------------------- def CalcHash(filename): hash = hashlib.sha256() with open(filename, "rb") as f: while True: block = f.read(4096) if not block: break hash.update(block) return hash.digest() # ---------------------------------------------------------------------- @contextlib.contextmanager def FileWriter(filename, mode): """\ Method that writes to a temporary location and only copies to the intended destination if there are changes. This prevents full rebuilds (which are triggered based on timestamps) on files that haven't changed. """ temp_filename = CurrentShell.CreateTempFilename() with open(temp_filename, mode) as f: yield f if not os.path.isfile(filename) or CalcHash( temp_filename) != CalcHash(filename): FileSystem.RemoveFile(filename) shutil.move(temp_filename, filename) else: FileSystem.RemoveFile(temp_filename) # ---------------------------------------------------------------------- this_dm.result = plugin.Generate( FileWriter, global_custom_structs, global_custom_enums, data, output_dir, this_dm.stream, ) if this_dm.result != 0: return this_dm.result return dm.result
def PreprocessContext(cls, context): # Augment all elements with their corresponding relational elements all_objects = [] all_elements = pickle.loads(context["pickled_elements"]) # Pass 1: Create objects for element in _EnumCompoundElements(all_elements): element.Object = Object.FromElement(element) all_objects.append(element.Object) # Pass 2: Relationships for element in _EnumCompoundElements(all_elements): if element.Parent: element.Object.Add(Relationship.FromChild(element)) for child in cls._EnumerateChildren( element, include_definitions=False, ): if isinstance(child, FundamentalElement): element.Object.Add( Fundamental( child, StringHelpers.ToSnakeCase(child.Name), child.TypeInfo, is_identity=False, is_mutable=child.mutable, is_index=child.index, is_unique=child.unique, ), ) elif isinstance(child, CompoundElement): # Nothing to do here, as the child will create a reference to # this element when created. pass elif isinstance(child, ExtensionElement): content = child.PositionalArguments[0] if content.startswith('"'): content = content[1:] if content.endswith('"'): content = content[:-1] element.Object.Add(content) elif isinstance(child, ReferenceElement): resolved_element = child.Resolve() if isinstance(resolved_element, FundamentalElement): element.Object.Add( Fundamental( child, StringHelpers.ToSnakeCase(child.Name), child.TypeInfo, is_identity=False, is_mutable=getattr(child, "mutable", False), is_index=getattr(child, "index", False), is_unique=getattr(child, "unique", False), ), ) elif isinstance(resolved_element, CompoundElement): element.Object.Add( Relationship.FromReference(child, resolved_element)) else: assert False, resolved_element elif isinstance(child, ListElement): assert isinstance(child.Reference, CompoundElement), child.Reference element.Object.Add( Relationship.FromReference(child, child.Reference)) else: assert False, child cls.AllElements = all_elements cls.AllObjects = all_objects return context
def CreateAdditionalDataItem(cls, dest_writer, name_var_name, source_var_name): temporary_element = cls.CreateTemporaryElement(name_var_name, "1") temporary_children_element = cls.CreateTemporaryElement("k", "+") return textwrap.dedent( """\ # The following types should be returned directly without additional conversion if isinstance({source_var_name}, (int, float, str, bool)): return {source_var_name} assert not isinstance({source_var_name}, list), {source_var_name} if not isinstance({source_var_name}, dict): {source_var_name} = {source_var_name}.__dict__ source_attribute_names = {source_var_name}.get("{attribute_names}", set()) attributes = OrderedDict() items = OrderedDict() for k, v in six.iteritems(source): if k.startswith("_"): continue if k in source_attribute_names: attributes[k] = v else: items[k] = v if len(items) == 1 and next(six.iterkeys(items)) == {source_var_name}.get("{fundamental_name}", None): return {simple_statement} result = {compound_statement} for k, v in six.iteritems(items): try: if isinstance(v, list): new_items = [] for index, child in enumerate(v): try: new_items.append(cls._CreateAdditionalDataItem("item", child)) except: _DecorateActiveException("Index {{}}".format(index)) {append_children} else: new_item = cls._CreateAdditionalDataItem(k, v) {append_child} except: _DecorateActiveException(k) return result """, ).format( source_var_name=source_var_name, attribute_names=cls.ATTRIBUTES_ATTRIBUTE_NAME, fundamental_name=cls.SIMPLE_ELEMENT_FUNDAMENTAL_ATTRIBUTE_NAME, simple_statement=StringHelpers.LeftJustify( dest_writer.CreateSimpleElement( temporary_element, "attributes", '{}[{}["{}"]]'.format(source_var_name, source_var_name, cls.SIMPLE_ELEMENT_FUNDAMENTAL_ATTRIBUTE_NAME), ), 4, ).strip(), compound_statement=dest_writer.CreateCompoundElement(temporary_element, "attributes").strip(), append_children=StringHelpers.LeftJustify( dest_writer.AppendChild(temporary_children_element, "result", dest_writer.CreateCollection(temporary_children_element, "new_items")), 12, ).strip(), append_child=StringHelpers.LeftJustify(dest_writer.AppendChild(cls.CreateTemporaryElement("k", "1"), "result", "new_item"), 8).strip(), )
def CreateCMakeFile( output_filename, data, output_name, input_filenames, cpp_wrapper_filename, generate_header_func, binary_version="1.0.0", ): output = [generate_header_func("# ")] output.append( textwrap.dedent( """\ cmake_minimum_required(VERSION 3.5.0) project({output_name} VERSION {binary_version} LANGUAGES CXX) set(CMAKE_MODULE_PATH "$ENV{{DEVELOPMENT_ENVIRONMENT_CMAKE_MODULE_PATH}}") set(_includes "$ENV{{INCLUDE}}") set(_libs "$ENV{{LIB}}") if(NOT WIN32) string(REPLACE ":" ";" CMAKE_MODULE_PATH "${{CMAKE_MODULE_PATH}}") string(REPLACE ":" ";" _includes "${{_includes}}") string(REPLACE ":" ";" _libs "${{_libs}}") endif() include(CppCommon OPTIONAL) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) if(CMAKE_CXX_COMPILER_ID MATCHES Clang) foreach(_flag IN ITEMS -Wno-missing-prototypes ) string(APPEND CMAKE_CXX_FLAGS " ${{_flag}}") endforeach() endif() add_library( {output_name} SHARED {filenames} ) target_include_directories({output_name} PRIVATE ${{_includes}}) target_link_directories({output_name} PRIVATE ${{_libs}}) """, ).format( output_name=output_name, binary_version=binary_version, filenames=StringHelpers.LeftJustify( Path("\n".join(input_filenames + [cpp_wrapper_filename])).as_posix(), 4).rstrip(), ), ) with open(output_filename, "w") as f: f.write("".join(output)) return 0
def Execute( cls, on_status_update, compiler, context, command_line, includes=None, excludes=None, verbose=False, ): execute_result = cls.ExecuteResult( test_result=None, test_output=None, test_time=None, ) coverage_start_time = time.time() coverage_output = OrderedDict() code_coverage_executor = cls._CodeCoverageExecutor() # ---------------------------------------------------------------------- def Impl(): # Instrument the binaries on_status_update("Instrumenting Binaries") # ---------------------------------------------------------------------- def Invoke(task_index, output_stream): output_filename = context["output_filenames"][task_index] return code_coverage_executor.PreprocessBinary( output_filename, output_stream) # ---------------------------------------------------------------------- sink = six.moves.StringIO() execute_result.CoverageResult = TaskPool.Execute( [ TaskPool.Task(output_filename, Invoke) for output_filename in context["output_filenames"] ], sink, verbose=True, ) coverage_output["Instrumenting Binaries"] = sink.getvalue() if execute_result.CoverageResult != 0: return # Start coverage coverage_output_filename = os.path.join( context["output_dir"], code_coverage_executor.DefaultFileName) on_status_update("Starting Coverage Monitor") sink = six.moves.StringIO() execute_result.CoverageResult = code_coverage_executor.StartCoverage( coverage_output_filename, sink) coverage_output["Starting Coverage Monitor"] = sink.getvalue() if execute_result.CoverageResult != 0: return # Execute the test(s) on_status_update("Testing") test_start_time = time.time() sink = six.moves.StringIO() execute_result.TestResult = code_coverage_executor.Execute( command_line, sink) execute_result.TestOutput = sink.getvalue() execute_result.TestTime = datetime.timedelta( seconds=(time.time() - test_start_time), ) if execute_result.TestResult != 0: return # Stop code coverage monitoring and extract the results on_status_update("Stopping Coverage Monitor") sink = six.moves.StringIO() execute_result.CoverageResult = code_coverage_executor.StopCoverage( sink) coverage_output["Stopping Coverage Monitor"] = sink.getvalue() if execute_result.CoverageResult != 0: return # Process the results output_names = [ os.path.basename(output_filename) for output_filename in context["output_filenames"] ] all_results = [None] * len(output_names) nonlocals = CommonEnvironment.Nonlocals( remaining=len(output_names), ) nonlocals_lock = threading.Lock() status_template = "Extracting Coverage Results ({} remaining)" on_status_update(status_template.format(nonlocals.remaining)) # ---------------------------------------------------------------------- def Invoke(task_index, output_stream): output_filename = context["output_filenames"][task_index] # This is a filename that can be used to specify includes and excludes. Note that this # does not correspond to an actual file, as we don't have that information available. mock_filter_filename = os.path.join( context["input"], os.path.splitext(os.path.basename(output_filename))[0], ) includes, excludes = CodeCoverageFilter.GetFilters( mock_filter_filename) this_result = code_coverage_executor.ExtractCoverageInfo( coverage_output_filename, output_filename, includes, excludes, output_stream, ) if not isinstance(this_result, tuple): return this_result covered, not_covered = this_result all_results[task_index] = (covered, not_covered) with nonlocals_lock: nonlocals.remaining -= 1 on_status_update( status_template.format(nonlocals.remaining)) return 0 # ---------------------------------------------------------------------- sink = six.moves.StringIO() execute_result.CoverageResult = TaskPool.Execute( [ TaskPool.Task(output_name, Invoke) for output_name in output_names ], sink, verbose=True, ) coverage_output["Extracting Coverage Results"] = sink.getvalue() if execute_result.CoverageResult != 0: return # Concatenate the results on_status_update("Finalizing Results") total_covered = 0 total_not_covered = 0 all_percentages = OrderedDict() for output_name, (covered, not_covered) in zip(output_names, all_results): total_covered += covered total_not_covered += not_covered result_blocks = covered + not_covered all_percentages[output_name] = ( None if not result_blocks else ((float(covered) / result_blocks) * 100.0), "{} of {} {} covered".format(covered, result_blocks, code_coverage_executor.Units), ) total_blocks = total_covered + total_not_covered execute_result.CoverageDataFilename = coverage_output_filename execute_result.CoveragePercentage = (float(total_covered) / total_blocks if total_blocks else 0.0) * 100.0 execute_result.CoveragePercentages = all_percentages # ---------------------------------------------------------------------- Impl() execute_result.CoverageOutput = "".join([ textwrap.dedent( """\ {} {} {} """, ).format( header, "-" * len(header), StringHelpers.LeftJustify(content.strip(), 4)) for header, content in six.iteritems(coverage_output) ], ) execute_result.CoverageTime = datetime.timedelta( seconds=(time.time() - coverage_start_time), ) # Subtract the time spent testing (if it exists) if execute_result.TestTime is not None: assert execute_result.CoverageTime >= execute_result.TestTime execute_result.CoverageTime -= execute_result.TestTime execute_result.TestTime = str(execute_result.TestTime) execute_result.CoverageTime = str(execute_result.CoverageTime) return execute_result
def EntryPoint( root_dir, output_stream=sys.stdout, ): with StreamDecorator(output_stream).DoneManager( line_prefix='', prefix="\nResults: ", suffix='\n', ) as dm: repositories = [] dm.stream.write( "\nSearching for repositories in '{}'...".format(root_dir)) with dm.stream.DoneManager(done_suffix=lambda: inflect.no( "repository", len(repositories)), ): for scm, directory in EnumSCMs(root_dir): if scm.Name != "Mercurial": continue repositories.append(directory) # Organize the repos dm.stream.write("Organizing...") with dm.stream.DoneManager(): repo_dict = OrderedDict() common_prefix = FileSystem.GetCommonPath(*repositories) common_prefix_len = len(common_prefix) for repository in repositories: suffix = repository[common_prefix_len:] parts = suffix.split(os.path.sep) repo_name = parts[-1] prefixes = parts[:-1] rd = repo_dict for prefix in prefixes: rd.setdefault(prefix, OrderedDict()) rd = rd[prefix] rd[repo_name] = repository # Write the content dm.stream.write("Writing TortoiseHg content...") with dm.stream.DoneManager(): filename = os.path.join(os.getenv("APPDATA"), "TortoiseHg", "thg-reporegistry.xml") assert os.path.isfile(filename), filename with open(filename, 'w') as f: # ---------------------------------------------------------------------- def GenerateContent(root, is_root): items = [] for k, v in six.iteritems(root): if isinstance(v, six.string_types): items.append( '<repo root="{}" shortname="{}" />\n'.format( v, os.path.basename(k), )) else: tag_name = "allgroup" if is_root else "group" items.append( textwrap.dedent("""\ <{tag_name} name="{name}"> {content} </{tag_name}> """).format( tag_name=tag_name, name=k, content=StringHelpers.LeftJustify( GenerateContent(v, False), 2).rstrip(), )) return ''.join(items) # ---------------------------------------------------------------------- f.write( textwrap.dedent("""\ <?xml version="1.0" encoding="UTF-8"?> <reporegistry> <treeitem> {} </treeitem> </reporegistry> """).format( StringHelpers.LeftJustify( GenerateContent(repo_dict, True).rstrip(), 4, skip_first_line=False, ))) return dm.result
def OnVariant(self, element): statements = [] new_types = [] for variation in element.Variations: if isinstance(variation, Elements.ReferenceElement): statement = "cls._{}_Item".format( ToPythonName(variation.Reference)) resolved_element = variation.Reference.Resolve() if isinstance( resolved_element, (Elements.CompoundElement, Elements.SimpleElement)): statement = '((lambda item: {}(item, process_additional_data=False, always_include_optional=False)), "{}")'.format( statement, resolved_element.DottedName) statements.append(statement) else: assert not isinstance(variation, (Elements.CompoundElement, Elements.SimpleElement)), variation new_types.append(variation) statements.append("cls._{}_Item".format( ToPythonName(variation))) python_name = ToPythonName(element) self._output_stream.write( textwrap.dedent( """\ # ---------------------------------------------------------------------- @classmethod def _{python_name}_Item(cls, item): for method_info in [ {statements} ]: if isinstance(method_info, tuple): potential_method, class_name = method_info else: potential_method = method_info class_name = None try: result = potential_method(item) if class_name is not None: {apply_class_name_statement} return result except: pass raise {exception_type}Exception("The value cannot be converted to any of the supported variations") """, ).format( python_name=python_name, statements=StringHelpers.LeftJustify( "\n".join([ "{},".format(statement) for statement in statements ]), 8).rstrip(), apply_class_name_statement=StringHelpers.LeftJustify( self._dest_writer.AppendChild( self._dest_writer.CreateTemporaryElement( '"{}"'.format( self._dest_writer. VARIANT_CLASS_TYPE_ATTRIBUTE_NAME), "1"), "result", "class_name", ), 16, ).rstrip(), exception_type="Serialize" if self._is_serializer else "Deserialize", ), ) self.Accept(new_types)
def Invoke( self, output_stream=sys.stdout, verbose=False, print_results=False, allow_exceptions=False, ): arg_strings = list(self.Args) debug_mode = False if len(arg_strings) > 1 and arg_strings[1].lower( ) == DEBUG_COMMAND_LINE_ARG: debug_mode = True del arg_strings[1] # Is there a request for verbose help? if any( arg.startswith(self.CommandLineArgPrefix) and arg[len(self.CommandLineArgPrefix):].lower() in [ "?", "help", "h", ] for arg in arg_strings): # Is the request for a specific method? if len(self.EntryPoints) > 1 and len(arg_strings) > 2: potential_method_name = arg_strings[1] else: potential_method_name = None return self.Usage( verbose=True, potential_method_name=potential_method_name, ) # Get the function to call if len(self.EntryPoints) == 1: # If there is only 1 entry point, don't make the user provide the name # on the command line entry_point = self.EntryPoints[0] arg_strings = arg_strings[1:] else: # The first arg is the entry point name if len(arg_strings) < 2: return self.Usage(verbose=verbose) name = arg_strings[1] name_lower = name.lower() arg_strings = arg_strings[2:] entry_point = next( (ep for ep in self.EntryPoints if ep.Name.lower() == name_lower), None) if entry_point is None: return self.Usage( error="'{}' is not a valid command".format(name), verbose=verbose, ) assert entry_point if debug_mode: output_stream.write( textwrap.dedent("""\ DEBUG INFO: {} """).format('\n'.join( [str(parameter) for parameter in entry_point.Parameters]))) return 1 # Read the arguments from a file if necessary if len(arg_strings) == 1 and arg_strings[0].startswith( self.ArgsInAFilePrefix): filename = os.path.join( os.getcwd(), arg_strings[0][len(self.ArgsInAFilePrefix):]) if not os.path.isfile(filename): return self.Usage( error="'{}' is not a valid filename".format(filename), verbose=verbose, ) arg_strings = [] with open(filename) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith('#'): continue arg_strings.append(line) # Parse the command line result = self._ParseCommandLine(entry_point, arg_strings) if isinstance(result, six.string_types): return self.Usage( error=result, verbose=verbose, ) kwargs = result if verbose: output_stream.write( textwrap.dedent("""\ INFO: Calling '{name}' was the arguments: {args} """).format( name=entry_point.Name, args='\n'.join([ " {k:<20} {v}".format( k="{}:".format(k), v=v, ) for k, v in six.iteritems(kwargs) ]), )) # Invoke the method try: result = entry_point(**kwargs) if print_results: if isinstance(result, types.GeneratorType): result = '\n'.join([ "{}) {}".format(index, str(item)) for index, item in enumerate(result) ]) output_stream.write( textwrap.dedent("""\ ** Result ** {} """).format(result)) result = 0 if result is None: result = 0 except UsageException as ex: result = self.Usage( error=str(ex), verbose=verbose, ) except ValidationException as ex: result = self.Usage( error=str(ex), verbose=verbose, ) except KeyboardInterrupt: result = -1 except: if allow_exceptions: raise if not getattr(sys.exc_info()[1], "_DisplayedException", False): import traceback output_stream.write("ERROR: {}".format( StringHelpers.LeftJustify(traceback.format_exc(), len("ERROR: ")))) result = -1 return result
def _GenerateClass(self, element): if getattr(element, "as_dictionary", False): return self._GenerateDictionary(element) attributes = [] statements = [] attribute_names = [] for child in self._enumerate_children_func( element, include_definitions=False, ): child_python_name = ToPythonName(child) resolved_child_python_name = ToPythonName(child.Resolve()) attribute_names.append(child.Name) is_compound_like = isinstance( child.Resolve(), (Elements.CompoundElement, Elements.SimpleElement)) # Note that we have to use getattr here, as Compound- and SimpleElements don't support IsAttribute if getattr(child, "IsAttribute", False): is_attribute = True assert not child.TypeInfo.Arity.IsCollection if child.TypeInfo.Arity.IsOptional: if hasattr(child, "default"): default_value = ', default_value_func=lambda: StringSerialization.DeserializeItem({}_TypeInfo, "{}")'.format( resolved_child_python_name, child.default) else: default_value = "" self.IncludeApplyOptionalAttribute = True statement_template = 'cls._ApplyOptionalAttribute(item, "{{name}}", attributes, cls.{{python_name}}, always_include_optional{})'.format( default_value) else: statement_template = textwrap.dedent( """\ attributes["{name}"] = cls.{python_name}( {get_child_statement}, ) """, ) statement = statement_template.format( name=child.Name, python_name=child_python_name, get_child_statement=StringHelpers.LeftJustify( self._source_writer.GetChild("item", child), 4).strip(), ) else: is_attribute = False if child.TypeInfo.Arity.Min == 0: if is_compound_like: statement = "lambda value: cls.{}(value, always_include_optional, process_additional_data)".format( child_python_name) else: statement = "cls.{}".format(child_python_name) if child.TypeInfo.Arity.Max == 1: self.IncludeApplyOptionalChild = True function_name = "_ApplyOptionalChild" if hasattr(child, "default"): default_value = ', default_value_func=lambda: StringSerialization.DeserializeItem({}_TypeInfo, "{}")'.format( resolved_child_python_name, child.default) else: default_value = "" else: self.IncludeApplyOptionalChildren = True function_name = "_ApplyOptionalChildren" default_value = "" statement = 'cls.{function_name}(item, "{name}", result, {statement}, always_include_optional{default_value})'.format( function_name=function_name, name=child.Name, statement=statement, default_value=default_value, ) else: if is_compound_like: extra_params = StringHelpers.LeftJustify( textwrap.dedent( """\ always_include_optional, process_additional_data """, ), 4, ) else: extra_params = "" statement = self._dest_writer.AppendChild( child, "result", textwrap.dedent( """\ cls.{python_name}( {get_child},{extra_params} ) """, ).format( python_name=ToPythonName(child), get_child=StringHelpers.LeftJustify( self._source_writer.GetChild( "item", child), 4).strip(), name=child.Name, extra_params=extra_params, ), ) (attributes if is_attribute else statements).append( textwrap.dedent( """\ # {name} try: {statement} except: _DecorateActiveException("{name}") """, ).format( name=child.Name, statement=StringHelpers.LeftJustify(statement, 4).strip(), ), ) if isinstance(element, Elements.SimpleElement): attribute_names.append(element.FundamentalAttributeName) statement = textwrap.dedent( """\ # <fundamental value> try: fundamental_value = {} except: _DecorateActiveException("value type") result = {} {} """, ).format( StringHelpers.LeftJustify( "{type_info}.{method_prefix}Item({python_name}__value__TypeInfo, {value}, **{serialize_args})" .format( type_info=self._type_info_serialization_name, method_prefix=self._method_prefix, python_name=ToPythonName(element), value=self._source_writer.GetChild( "item", self._source_writer.CreateTemporaryElement( '"{}"'.format( element.FundamentalAttributeName), "1"), is_simple_schema_fundamental=True, ), serialize_args=self._custom_serialize_item_args, ), 4, ), self._dest_writer.CreateSimpleElement( element, "attributes" if attributes else None, "fundamental_value"), "".join(statements).strip(), ) else: statement = textwrap.dedent( """\ result = {} {} """, ).format( self._dest_writer.CreateCompoundElement( element, "attributes" if attributes else None).strip(), "".join(statements).strip()) python_name = ToPythonName(element) validation_statement_template = textwrap.dedent( """\ {}_TypeInfo.ValidateItem( {{}}, recurse=False, require_exact_match=not process_additional_data, ) """, ).format(python_name) if self._is_serializer: prefix = validation_statement_template.format("item") suffix = "" else: prefix = "" suffix = textwrap.dedent( """\ else: cls._RejectAdditionalData( item, exclude_names=[{attribute_names}], ) {validation_statement} """, ).format( attribute_names=", ".join([ '"{}"'.format(attribute_name) for attribute_name in attribute_names ]), validation_statement=validation_statement_template.format( "result"), ) self._output_stream.write( textwrap.dedent( """\ # ---------------------------------------------------------------------- @classmethod def _{python_name}_Item(cls, item, always_include_optional, process_additional_data): {prefix}{prefix_whitespace}{attributes_decl}{attributes}{statement} # Additional data if process_additional_data: cls._ApplyAdditionalData( item, result, exclude_names={{{attribute_names}}}, ) {suffix} return result """, ).format( python_name=python_name, prefix=StringHelpers.LeftJustify( "{}\n\n".format(prefix.strip()) if prefix else prefix, 4), prefix_whitespace=" " if prefix else "", suffix=StringHelpers.LeftJustify( "\n{}\n".format(suffix.strip()) if suffix else suffix, 4), attributes_decl="" if not attributes else "attributes = OrderedDict()\n\n ", attributes="" if not attributes else "{}\n\n ".format( StringHelpers.LeftJustify("".join(attributes), 4).strip()), statement=StringHelpers.LeftJustify(statement, 4).strip(), attribute_names=", ".join([ '"{}"'.format(attribute_name) for attribute_name in attribute_names ]), ), )
def Execute( root_dir, output_dir, mode=None, debug_only=False, release_only=False, output_stream=sys.stdout, verbose=False, ): """Recursively calls Build files with the desired mode(s)""" assert os.path.isdir(root_dir), root_dir assert output_dir modes = mode or [ "clean", "build", ]; del mode assert output_stream with StreamDecorator(output_stream).DoneManager( line_prefix='', prefix="\nResults: ", suffix='\n', ) as dm: build_infos = _GetBuildInfos(root_dir, dm.stream) if not build_infos: return dm.result # Find all the build files that have configurations that we can process build_configurations = [] dm.stream.write("Processing build files...") with dm.stream.DoneManager( done_suffix=lambda: "{} found".format(inflect.no("configuration", len(build_configurations))), ) as this_dm: # ---------------------------------------------------------------------- def GetSupportedConfigurations(configurations): # If there is a configuration that indicates completeness, execute that # and skip everything else. if COMPLETE_CONFIGURATION_NAME in configurations: yield COMPLETE_CONFIGURATION_NAME return for config in build_configurations: config_lower = config.lower() if ( (debug_only and "debug" in config_lower) or (release_only and "release" in config_lower) or (not debug_only and not release_only) ): yield config # ---------------------------------------------------------------------- for build_info in build_infos: if not build_info.configuration.Configurations: build_configurations.append(( build_info.filename, build_info.configuration, None, )) else: for config in GetSupportedConfigurations(build_info.configuration.Configurations): build_configurations.append(( build_info.filename, build_info.configuration, config, )) if not build_configurations: return dm.result dm.stream.write('\n') for mode_index, mode in enumerate(modes): dm.stream.write("Invoking '{}' ({} of {})...".format( mode, mode_index + 1, len(modes), )) with dm.stream.DoneManager() as mode_dm: for build_index, (build_filename, config, configuration) in enumerate(build_configurations): mode_dm.stream.write("Processing '{}'{} ({} of {})...".format( build_filename, " - '{}'".format(configuration) if configuration else '', build_index + 1, len(build_configurations), )) with mode_dm.stream.DoneManager() as build_dm: build_output_dir = os.path.join(output_dir, config.SuggestedOutputDirLocation, configuration or "Build") FileSystem.MakeDirs(build_output_dir) command_line = 'python "{build_filename}" {mode}{configuration}{output_dir}' \ .format( build_filename=build_filename, mode=mode, configuration=' "{}"'.format(configuration) if configuration else '', output_dir=' "{}"'.format(build_output_dir) if config.RequiresOutputDir else '', ) build_dm.result, output = Process.Execute(command_line) # It is possible that the cleaning process deleted the output directory. Recreate it # if necessary to store the log file. FileSystem.MakeDirs(build_output_dir) with open(os.path.join(build_output_dir, BUILD_LOG_TEMPLATE.format(mode=mode)), 'w') as f: f.write(output) if build_dm.result != 0: build_dm.stream.write(output) elif verbose: build_dm.stream.write(StringHelpers.LeftJustify("INFO: {}".format(output), len("INFO: "))) return dm.result
def DoneManager( self, line_prefix=" ", prefix=None, # string or def Func() -> string suffix=None, # string or def Func() -> string done_suffix=None, # string or def Func() -> string done_suffixes=None, # List of string or def Func() -> string display=True, # Display done information; will sometimes be False for nested DoneManagers created to manage error propagation display_result=True, # Display the result display_time=True, # Display the time delta display_exceptions=True, # Display exception information display_exception_callstacks=True, # Display exception callstack info suppress_exceptions=False, # Do not let exception propagate associated_stream=None, associated_streams=None, # Streams that should be adjusted in conjunction with this stream. Most of the time, this is used # to manage verbose streams that are aligned with status streams, where the status stream is self # and the verbose stream content is interleaved with it. # # Example: # import sys # # from StringIO import StringIO # from CommonEnvironment.StreamDecorator import StreamDecorator # # sink = StringIO() # # verbose_stream = StreamDecorator(sink) # status_stream = StreamDecorator([ sys.stdout, verbose_stream, ]) # # status_stream.write("0...") # with status_stream.DoneManager(associated_stream=verbose_stream) as ( dm1, verbose_stream1 ): # verbose_stream1.write("Verbose 0\n----") # # dm1.stream.write("1...") # with dm1.stream.DoneManager(associated_stream=verbose_stream1) as ( dm2, verbose_stream2 ): # verbose_stream2.write("Verbose 1\n----") # # dm2.stream.write("2...") # with dm2.stream.DoneManager(associated_stream=verbose_stream2) as ( dm3, verbose_stream3 ): # verbose_stream3.write("Verbose 2\n----") # # dm3.stream.write("3...") # with dm3.stream.DoneManager(associated_stream=verbose_stream3) as ( dm4, verbose_stream4 ): # verbose_stream4.write("Verbose 3\n----") # verbose_stream4.flush() # # sys.stdout.write("\n**\n{}\n**\n".format(sink.getvalue())) ): """Displays done information upon completion.""" assert display_exceptions or not suppress_exceptions, "It isn't wise to not display exceptions while also suppressing them" # ---------------------------------------------------------------------- def ToFunctor(value): if callable(value): return value if value is None: value = '' return lambda: value # ---------------------------------------------------------------------- prefix_func = ToFunctor(prefix) suffix_func = ToFunctor(suffix) if not display: line_prefix = '' nonlocals = Nonlocals(time_delta=None) info = self._DoneManagerInfo(self, line_prefix) done_suffixes_funcs = [ ToFunctor(ds) for ds in (done_suffixes or []) ] if done_suffix: done_suffixes_funcs.append(ToFunctor(done_suffix)) if display_time: done_suffixes_funcs.insert(0, lambda: str(nonlocals.time_delta)) if display_result: done_suffixes_funcs.insert(0, lambda: str(info.result)) # ---------------------------------------------------------------------- def Cleanup(): assert info.result is not None if display: prefix = prefix_func() suffix = suffix_func() suffixes = [] for done_suffix in done_suffixes_funcs: result = done_suffix() if result: suffixes.append(result) if suffixes: content = ', '.join(suffixes) if prefix.strip(): content = "({})".format(content) elif not line_prefix: pass else: # Single-line results content = "DONE! ({})".format(content) else: content = '' self.write("{}{}{}\n".format( prefix, content, suffix, )) self.flush() # Propogate the result if info.result != 0: for index, dm in enumerate(info.Enumerate()): if index == 0: continue if ( dm.result != info.result and (dm.result == 0 or (dm.result > 0 and info.result < 0)) ): dm.result = info.result # ---------------------------------------------------------------------- with CallOnExit(Cleanup): start_time = time.time() try: associated_streams = associated_streams or [] if associated_stream: associated_streams.append(associated_stream) if not associated_streams: yield info else: yield tuple([ info, ] + [ StreamDecorator( stream, line_prefix=' ' * len(line_prefix), one_time_prefix='\n', one_time_suffix="\n<flush>", is_associated_stream=True, ) for stream in associated_streams ]) if info.result is None: info.result = 0 except Exception: if info.result is None or info.result >= 0: info.result = -1 if display_exceptions: ex = sys.exc_info()[1] if ( not getattr(ex, "_DisplayedException", False) and not getattr(info.stream, "IsAssociatedStream", False) ): ex._DisplayedException = True if display_exception_callstacks: import traceback info.stream.write("ERROR: {}\n".format(StringHelpers.LeftJustify(traceback.format_exc(), len("ERROR: ")).rstrip())) else: info.stream.write("ERROR: {}\n".format(StringHelpers.LeftJustify(str(ex), len("ERROR: ")).rstrip())) if not suppress_exceptions: raise finally: current_time = time.time() # There is a bug (and I have seen it) where the value calculated # in the past will be greater than the value calculated now. This is # wrong in theory, but apparently there is a BIOS bug that causes the # behavior on multicore machines (I have a hunch that virtualized machines # contribute to the problem as well). More info at http://bytes.com/topic/python/answers/527849-time-clock-going-backwards. # Regardless, asserting here is causing problems and this method is # only used for scripts. If we encounter the scenario, populate with # bogus data. if start_time > current_time: # This is a total lie, but hopefully the value is unique enough to # generate a double take. This is preferable to causing a long- # running process to fail. current_time = start_time + (12 * 60 * 60) + (34 * 60) + 56 # 12:34:56 assert current_time >= start_time, (current_time, start_time) nonlocals.time_delta = str(datetime.timedelta(seconds=current_time - start_time))
def CreateAdditionalDataItem(cls, dest_writer, name_var_name, source_var_name): temporary_element = cls.CreateTemporaryElement( "{}.tag".format(source_var_name), "1") return textwrap.dedent( """\ attributes = OrderedDict() for k, v in six.iteritems({source_var_name}.attrib): if k.startswith("_"): continue attributes[k] = v if {source_var_name}.text and {source_var_name}.text.strip() and not {source_var_name}: return {simple_element} result = {compound_statement} for child_name, child_or_children in cls._GenerateAdditionalDataChildren({source_var_name}, set()): try: if isinstance(child_or_children, list): new_items = [] for index, child in enumerate(child_or_children): try: new_items.append(cls._CreateAdditionalDataItem("{collection_item_name}", child)) except: _DecorateActiveException("Index {{}}".format(index)) if child_name is None: result = new_items break {append_children} else: assert child_name is not None new_item = cls._CreateAdditionalDataItem(child_name, child_or_children) {append_child} except: _DecorateActiveException(child_name) return result """, ).format( source_var_name=source_var_name, collection_item_name=Plugin.COLLECTION_ITEM_NAME, compound_statement=dest_writer.CreateCompoundElement( temporary_element, "attributes").strip(), simple_element=StringHelpers.LeftJustify( dest_writer.CreateSimpleElement( temporary_element, "attributes", "{}.text".format(source_var_name)), 4).strip(), append_children=StringHelpers.LeftJustify( dest_writer.AppendChild( cls.CreateTemporaryElement("child_name", "+"), "result", "new_items"), 12).strip(), append_child=StringHelpers.LeftJustify( dest_writer.AppendChild( cls.CreateTemporaryElement("child_name", "1"), "result", "new_item"), 8).strip(), )
def __init__( self, identity, unique_name, name, element_or_source_line_column_tuple, ): super(Object, self).__init__(element_or_source_line_column_tuple) self.Identity = identity self.SingularName = name self.PluralName = inflect.plural(name) self.UniqueName = unique_name self.SingularPascalName = StringHelpers.ToPascalCase(self.SingularName) self.PluralPascalName = StringHelpers.ToPascalCase(self.PluralName) self.UniquePascalName = StringHelpers.ToPascalCase(self.UniqueName) self.SingularSnakeName = StringHelpers.ToSnakeCase(self.SingularName) self.PluralSnakeName = StringHelpers.ToSnakeCase(self.PluralName) self.UniqueSnakeName = StringHelpers.ToSnakeCase(self.UniqueName) self.constraints = [] self.children = [] # Add the identity info self.Add( Fundamental( (_script_name, inspect.currentframe().f_lineno, 0), "id", IntTypeInfo(min=0) if self.Identity == self.IdentityType.Integer else GuidTypeInfo(), is_identity=True, is_mutable=False, is_index=True, is_unique=True, ), ) if self.Identity in [ self.IdentityType.Simple, self.IdentityType.Deletable ]: self.Add( Fundamental( (_script_name, inspect.currentframe().f_lineno, 0), "created", DateTimeTypeInfo(), is_identity=True, is_mutable=False, is_index=False, is_unique=False, ), ) if self.Identity == self.IdentityType.Deletable: for name in ["deleted", "restored"]: self.Add( Fundamental( (_script_name, inspect.currentframe().f_lineno, 0), name, DateTimeTypeInfo(arity="?", ), is_identity=True, is_mutable=False, is_index=False, is_unique=False, ), )