def CommandLineCleanOutputFilename(output_filename, output_stream): output_stream = StreamDecorator(output_stream) if not os.path.isfile(output_filename): output_stream.write("'{}' does not exist.\n".format(output_filename)) else: output_stream.write("Removing '{}'...".format(output_filename)) with output_stream.DoneManager(): FileSystem.RemoveFile(output_filename) return 0
def Clean(cls, context, optional_output_stream): """ Handles the complexities associated with cleaning previously generated output, ultimately invoking _CleanImpl. """ assert context output_stream = StreamDecorator(optional_output_stream) output_stream.write( cls._GetStatusText("Cleaning", context, cls.GetInputItems(context))) with output_stream.DoneManager() as dm: dm.result = cls._CleanImpl(context, dm.stream) or 0 return dm.result
def _CleanImplEx(cls, context, output_stream): output_stream = StreamDecorator(output_stream) input_items = set(cls.GetInputItems(context)) for output_filename in context["output_filenames"]: if output_filename in input_items: continue if os.path.isfile(output_filename): output_stream.write("Removing '{}'...".format(output_filename)) with output_stream.DoneManager(): FileSystem.RemoveFile(output_filename) return super(MultipleOutputMixin, cls)._CleanImplEx(context, output_stream)
def _Impl( display_sentinel, json_filename, result_filename, first, output_stream, method_name, parser, ): output_stream = StreamDecorator( output_stream, line_prefix=display_sentinel, ) with open(json_filename) as f: try: data = parser(f.read(), is_root=True) except Exception as ex: output_stream.write("ERROR: {} ({})\n".format(str(ex), ex.stack)) return -1 output_stream.write("Parsing dependencies...") with output_stream.DoneManager(): dependencies = ActivationData.Load(None, None, None).PrioritizedRepositories has_config_specific = False output_stream.write("Validating...") with output_stream.DoneManager() as dm: for index, repository_info in enumerate(dependencies): dm.stream.write("Processing '{}' ({} of {})...".format( repository_info.Name, index + 1, len(dependencies), )) with dm.stream.DoneManager() as this_dm: with Utilities.CustomMethodManager(os.path.join(repository_info.Root, Constants.HOOK_ENVIRONMENT_CUSTOMIZATION_FILENAME), method_name) as method: if not method: continue args = OrderedDict([ ( "data", data ), ( "output_stream", this_dm.stream ), ]) # Get the method args to see if a configuration is requried func_code = six.get_function_code(method) if "configuration" in func_code.co_varnames[:func_code.co_argcount]: args["configuration"] = repository_info.Configuration has_config_specific = True elif not first: # Don't call a config-agnostic method more than once continue try: this_dm.result = Interface.CreateCulledCallable(method)(args) or 0 except Exception as ex: this_dm.stream.write(StringHelpers.LeftJustify( "ERROR: {}\n".format(str(ex).rstrip()), len("ERROR: "), )) this_dm.result = -1 with open(result_filename, 'w') as f: f.write('-1' if dm.result != 0 else '1' if has_config_specific else '0') return dm.result
def CreatePluginMap( dynamic_plugin_architecture_environment_key, plugins_dir, output_stream, ): """ Loads all plugins specified by the given environment key or that reside in the provided plugins_dir. Returns dict: { "<plugin name>" : PluginInfo(), ... } """ plugin_mods = [] if os.getenv(dynamic_plugin_architecture_environment_key): assert os.getenv("DEVELOPMENT_ENVIRONMENT_FUNDAMENTAL") sys.path.insert(0, os.getenv("DEVELOPMENT_ENVIRONMENT_FUNDAMENTAL")) with CallOnExit(lambda: sys.path.pop(0)): from RepositoryBootstrap.SetupAndActivate import DynamicPluginArchitecture plugin_mods = list( DynamicPluginArchitecture.EnumeratePlugins( dynamic_plugin_architecture_environment_key)) elif os.path.isdir(plugins_dir): for filename in FileSystem.WalkFiles( plugins_dir, include_file_base_names=[ lambda name: name.endswith("Plugin"), ], include_file_extensions=[ ".py", ], recurse=True, ): name = os.path.splitext(os.path.basename(filename))[0] plugin_mods.append(imp.load_source(name, filename)) else: raise Exception( "'{}' is not a valid environment variable and '{}' is not a valid directory" .format( dynamic_plugin_architecture_environment_key, plugins_dir, )) plugins = OrderedDict() # ---------------------------------------------------------------------- def GetPluginImpl(plugin_name, raise_on_error=True): if plugin_name not in plugins: if raise_on_error: raise Exception( "'{}' is not a valid plugin".format(plugin_name)) return None return plugins[plugin_name] # ---------------------------------------------------------------------- warning_stream = StreamDecorator( output_stream, line_prefix="WARNING: ", suffix='\n', ) with CallOnExit(lambda: warning_stream.flush(force_suffix=True)): if not plugin_mods: warning_stream.write("No plugins were found.\n") for plugin_mod in plugin_mods: obj = getattr(plugin_mod, "Plugin", None) if obj is None: warning_stream.write( "The module defined at '{}' does not contain a 'Plugin' class.\n" .format(plugin_mod.__file__)) continue # Dynamically add the method GetPlugin to the plugin object; this will allow # a plugin to query for other plugins. This is a bit wonky, but it works with # the plugin architecture where most of the plugins are static objects. obj.GetPlugin = staticmethod(GetPluginImpl) obj = obj() if not obj.IsValidEnvironment(): warning_stream.write( "The plugin '{}' is not valid within this environment ({}).\n" .format( obj.Name, plugin_mod.__file__, )) continue plugins[obj.Name] = PluginInfo(plugin_mod, obj) return plugins
def _InvokeImpl( cls, invoke_reason, context, status_stream, verbose_stream, verbose, ): # If the file is being invoked as a test file, measure the file under test # rather than the test itself. filename = context["input"] try: filename = cls.TestToItemName(filename) except: pass assert os.path.isfile(filename), filename if os.path.basename(filename) == "__init__.py" and os.path.getsize( filename) == 0: return 0 # Create the lint file configuration_file = os.getenv( cls.CONFIGURATION_ENVIRONMENT_VAR_NAME) or os.path.join( _script_dir, "PythonVerifier.default_configuration") assert os.path.isfile(configuration_file), configuration_file # Write the python script that invokes the linter temp_filename = CurrentShell.CreateTempFilename(".py") with open(temp_filename, 'w') as f: f.write( textwrap.dedent("""\ import sys from pylint import lint lint.Run([ r"--rcfile={config}", r"--msg-template={{path}}({{line}}): [{{msg_id}}] {{msg}}", r"{filename}", ]) """).format( config=configuration_file, filename=filename, )) with CallOnExit(lambda: FileSystem.RemoveFile(temp_filename)): # Run the generated file command_line = 'python "{}"'.format(temp_filename) sink = six.moves.StringIO() output_stream = StreamDecorator([ sink, verbose_stream, ]) regex_sink = six.moves.StringIO() Process.Execute(command_line, StreamDecorator([ regex_sink, output_stream, ])) regex_sink = regex_sink.getvalue() result = 0 # Extract the results match = re.search( r"Your code has been rated at (?P<score>[-\d\.]+)/(?P<max>[\d\.]+)", regex_sink, re.MULTILINE, ) if not match: result = -1 else: score = float(match.group("score")) max_score = float(match.group("max")) assert max_score != 0.0 # Don't measure scores for files in Impl directories is_impl_file = os.path.basename(filename).endswith("Impl") if is_impl_file and not context["explicit_passing_score"]: passing_score = None else: passing_score = context["passing_score"] output_stream.write( textwrap.dedent("""\ Score: {score} (out of {max_score}) Passing Score: {passing_score}{explicit} """).format( score=score, max_score=max_score, passing_score=passing_score, explicit=" (explicitly provided)" if context["explicit_passing_score"] else '', )) if passing_score is not None and score < passing_score: result = -1 if result != 0 and not verbose: status_stream.write(sink.getvalue()) return result
def _Invoke(cls, context, status_stream, verbose): """Handles the complexities of compiler invocation, ultimately calling _InvokeImpl.""" assert context status_stream = StreamDecorator(status_stream) invoke_reason = cls._GetInvokeReason( context, StreamDecorator(status_stream if verbose else None)) if invoke_reason is None: status_stream.write("No changes were detected.\n") return 0 input_items = cls.GetInputItems(context) assert input_items status_stream.write( cls._GetStatusText(cls.InvokeVerb, context, input_items)) with status_stream.DoneManager() as dm: if verbose: output_items = cls.GetOutputItems(context) if "display_name" in context or len(input_items) == 1: indentation = 4 else: indentation = 8 verbose_stream = StreamDecorator( dm.stream, prefix=StringHelpers.LeftJustify( textwrap.dedent( # <Wrong hanging indentation> pylint: disable = C0330 """\ ======================================== VERBOSE Output {} -> {} ======================================== """ ).format( '\n'.join(input_items), StringHelpers.LeftJustify( '\n'.join(output_items) if output_items else "[None]", 4), ), 2, skip_first_line=False, ), suffix='\n', line_prefix=' ' * indentation, ) status_stream = verbose_stream else: status_stream = dm.stream verbose_stream = StreamDecorator(None) dm.result = cls._InvokeImpl( invoke_reason, context, status_stream, verbose_stream, verbose, ) or 0 if dm.result >= 0: cls._PersistContext(context) return dm.result
def Usage( self, error=None, error_stream=sys.stderr, verbose=False, potential_method_name=None, ): error_stream.write( textwrap.dedent("""\ {desc}{prefix} Usage: """).format( desc=StringHelpers.Wrap(self.ScriptDescription, MAX_COLUMN_WIDTH), prefix='' if not self.ScriptDescriptionPrefix else "\n\n{}".format(self.ScriptDescriptionPrefix), )) indented_stream = StreamDecorator(error_stream, line_prefix=" ") # Narrow the list down if help was requested for a single method entry_points = self.EntryPoints if len(self.EntryPoints) > 1 and len(self.Args) >= 2: potential_method_name = self.Args[1].lower() for entry_point in entry_points: if entry_point.Name.lower() == potential_method_name: entry_points = [ entry_point, ] break # Display a single item or multiple items if len(entry_points) == 1: standard, verbose_desc = self._GenerateUsageInformation( entry_points[0]) # Add the method name if necessary if len(self.EntryPoints) > 1: if '\n' in standard: standard = "\n {}{}".format( entry_points[0].Name, standard, ) else: standard = "{} {}".format( entry_points[0].Name, standard, ) if verbose: standard = "{}\n\n{}".format(standard, verbose_desc) indented_stream.write(" {} {}\n\n".format( self.ScriptName, StringHelpers.LeftJustify(standard, 4), )) else: # ---------------------------------------------------------------------- def FormatInlineFuncDesc(content): initial_whitespace = 37 assert MAX_COLUMN_WIDTH > initial_whitespace content = StringHelpers.Wrap( content, MAX_COLUMN_WIDTH - initial_whitespace) return StringHelpers.LeftJustify(content, initial_whitespace) # ---------------------------------------------------------------------- indented_stream.write( textwrap.dedent("""\ {script_name} <command> [args] Where '<command>' can be one of the following: ---------------------------------------------- """).format(script_name=self.ScriptName, )) for entry_point in entry_points: indented_stream.write(" - {name:<30} {desc}\n".format( name=entry_point.Name, desc=FormatInlineFuncDesc(entry_point.Description), )) indented_stream.write('\n') for entry_point in entry_points: intro = "When '<command>' is '{}':".format(entry_point.Name) standard, verbose_desc = self._GenerateUsageInformation( entry_point) # Insert the function name as an argument if '\n' in standard: multi_line_args = True standard = " {}{}".format( entry_point.Name, StringHelpers.LeftJustify(standard, 4)) else: multi_line_args = False standard = "{} {}".format(entry_point.Name, standard) if verbose: standard = "{}\n\n{}".format( standard, StringHelpers.LeftJustify(verbose_desc, 4, skip_first_line=False), ) indented_stream.write( textwrap.dedent("""\ {intro} {sep} {script_name}{newline}{standard} """).format( intro=intro, sep='-' * len(intro), script_name=self.ScriptName, newline='\n' if multi_line_args else ' ', standard=standard, )) if self.ScriptDescriptionSuffix: error_stream.write("\n{}\n".format( self.ScriptDescriptionSuffix.strip())) if not verbose: error_stream.write( textwrap.dedent("""\ Run "{script_name} {prefix}?" for additional information. """).format( script_name=self.ScriptName, prefix=self.CommandLineArgPrefix, )) if error: error = "\n\nERROR: {}\n".format( StringHelpers.LeftJustify(error, len("ERROR: "))) try: import colorama colorama.init(autoreset=True) error_stream = sys.stderr error_stream.write("{}{}{}".format( colorama.Fore.RED, colorama.Style.BRIGHT, error, )) except ImportError: error_stream.write(error) return -1
def Generate( cls, simple_schema_generator, invoke_reason, input_filenames, output_filenames, name, elements, include_indexes, status_stream, verbose_stream, verbose, root_name, process_additional_data, ): assert len(output_filenames) == 1 output_filename = output_filenames[0] del output_filenames include_map = cls._GenerateIncludeMap(elements, include_indexes) include_dotted_names = set(six.iterkeys(include_map)) top_level_elements = [ element for element in elements if element.Parent is None and not element.IsDefinitionOnly and element.DottedName in include_map ] status_stream.write("Creating '{}'...".format(output_filename)) with status_stream.DoneManager() as dm: with open(output_filename, "w") as f: f.write( textwrap.dedent( """\ <?xml version="1.0"?> <xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema"> <!-- {} --> """, ).format( cls._GenerateFileHeader( line_break= "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-", filename_prefix="<SimpleSchemaGenerator>/", ), ), ) # XML only always 1 root element. If there are multiple top-level elements # defined by the schema, unite them under a common parent. if len(top_level_elements) > 1: f.write( textwrap.dedent( """\ <xsd:element name="{}"> <xsd:complexType> <xsd:sequence> """, ).format(root_name or name), ) element_stream = StreamDecorator( f, line_prefix=" ", ) # ---------------------------------------------------------------------- def Cleanup(): f.write( textwrap.dedent( """\ </xsd:sequence> </xsd:complexType> </xsd:element> """, ), ) # ---------------------------------------------------------------------- else: element_stream = f # ---------------------------------------------------------------------- def Cleanup(): f.write("\n") # ---------------------------------------------------------------------- with CallOnExit(Cleanup): for element in top_level_elements: element_stream.write( '<xsd:element name="{name}" type="_{type}" />\n'. format( name=element.Name, type=element.Resolve().DottedName, ), ) fundamental_type_info_visitor = _FundamentalTypeInfoVisitor() # ---------------------------------------------------------------------- def GetBaseTypeName( element, item_suffix=True, ): name = getattr(element, "_xsd_base_type", None) if name is None: name = "_{}".format(element.DottedName) if item_suffix: name = "{}_Item".format(name) return name # ---------------------------------------------------------------------- class Visitor(Elements.ElementVisitor): # ---------------------------------------------------------------------- @staticmethod @Interface.override def OnFundamental(element): content = fundamental_type_info_visitor.Accept( element.TypeInfo) if content.startswith("<xsd:restriction"): f.write( textwrap.dedent( """\ <xsd:simpleType name="_{}_Item"> {} </xsd:simpleType> """, ).format( element.DottedName, StringHelpers.LeftJustify( content, 2).strip()), ) else: element._xsd_base_type = content # ---------------------------------------------------------------------- @staticmethod @Interface.override def OnCompound(element): # Process the element after all children have been processed pass # ---------------------------------------------------------------------- @staticmethod @Interface.override def OnCompound_VisitedChildren(element): attributes = [] elements = [] for child in cls._EnumerateChildren( element, include_definitions=False, ): if getattr(child, "IsAttribute", False): attributes.append( textwrap.dedent( """\ <xsd:attribute name="{name}" use="{use}" type="{type}"{default} /> """, ).format( name=child.Name, use="optional" if child.TypeInfo.Arity.IsOptional else "required", type=GetBaseTypeName( child.Resolve()), default="" if not hasattr(child, "default") else ' default="{}"'.format( child.default), ), ) else: elements.append( textwrap.dedent( """\ <xsd:element name="{name}" type="_{type}" minOccurs="{min}" maxOccurs="1"{default} /> """, ).format( name=child.Name, type=child.DottedName, min="0" if child.TypeInfo.Arity.Min == 0 else "1", default="" if not hasattr(child, "default") else ' default="{}"'.format( child.default), ), ) element_process_additional_data = getattr( element, "process_additional_data", None) if element_process_additional_data is None: element_process_additional_data = process_additional_data if element_process_additional_data: elements.append( '<xsd:any minOccurs="0" maxOccurs="unbounded" processContents="skip" />\n' ) content = textwrap.dedent( """\ <xsd:sequence> {elements} </xsd:sequence> {attributes} """, ).format( elements=StringHelpers.LeftJustify( "".join(elements), 2).strip(), attributes="".join(attributes).strip(), ) f.write( textwrap.dedent( """\ <xsd:complexType name="_{name}_Item"{mixed}> {content} </xsd:complexType> """, ).format( name=element.DottedName, content=StringHelpers.LeftJustify( content, 2).strip(), mixed=' mixed="true"' if element_process_additional_data else "", ), ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def OnSimple(element): # Process the element after all children have been processed pass # ---------------------------------------------------------------------- @staticmethod @Interface.override def OnSimple_VisitedChildren(element): content = fundamental_type_info_visitor.Accept( element.TypeInfo.Items[ element.FundamentalAttributeName]) if content.startswith("<xsd:restriction"): f.write( textwrap.dedent( """\ <xsd:simpleType name="_{}_Item_content"> {} </xsd:simpleType> """, ).format( element.DottedName, StringHelpers.LeftJustify( content.strip(), 2)), ) content = "_{}_Item_content".format( element.DottedName) f.write( textwrap.dedent( """\ <xsd:complexType name="_{name}_Item"> <xsd:simpleContent> <xsd:extension base="{base}"> {attributes} </xsd:extension> </xsd:simpleContent> </xsd:complexType> """, ).format( name=element.DottedName, base=content, attributes=StringHelpers.LeftJustify( "".join([ textwrap.dedent( """\ <xsd:attribute name="{name}" use="{use}" type="{type}" /> """, ).format( name=attribute.Name, use="optional" if attribute.TypeInfo.Arity. IsOptional else "required", type=GetBaseTypeName( attribute.Resolve()), ) for attribute in element.Attributes ], ), 6, ).rstrip(), ), ) # ---------------------------------------------------------------------- @classmethod @Interface.override def OnVariant(this_cls, element): # Xsd makes it difficult to work with variants. We could # use <xsd:alternative...>, but the test clause is problematic # for the arbitrary types that can be expressed by the different # Element types. Bail on the authentication of variants by # allowing anything; this means that the variant will need to # be validate by other means. return this_cls.OnAny(element) # ---------------------------------------------------------------------- @staticmethod @Interface.override def OnReference(element): # References don't need to be added, as they will be resolved inline. pass # ---------------------------------------------------------------------- @staticmethod @Interface.override def OnList(element): f.write( textwrap.dedent( """\ <xsd:complexType name="_{name}_Item"> <xsd:complexContent> <xsd:extension base="_{type}" /> </xsd:complexContent> </xsd:complexType> """, ).format( name=element.DottedName, type=element.Reference.Resolve().DottedName, ), ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def OnAny(element): f.write( textwrap.dedent( """\ <xsd:complexType name="_{}_Item" mixed="true"> <xsd:sequence> <xsd:any minOccurs="0" processContents="skip" /> </xsd:sequence> </xsd:complexType> """, ).format(element.DottedName), ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def OnCustom(element): raise Exception("CustomElements are not supported") # ---------------------------------------------------------------------- @staticmethod @Interface.override def OnExtension(element): raise Exception("ExtensionElements are not supported") # ---------------------------------------------------------------------- Visitor().Accept( elements, include_dotted_names=include_dotted_names, ) # ---------------------------------------------------------------------- def OnElement(element): typ = GetBaseTypeName(element.Resolve()) if element.TypeInfo.Arity.Max == 1: f.write( textwrap.dedent( """\ <xsd:complexType name="_{name}"> <xsd:{type}Content> <xsd:extension base="{base}" /> </xsd:{type}Content> </xsd:complexType> """, ).format( type="simple" if isinstance( element.Resolve(), Elements.FundamentalElement) else "complex", name=element.DottedName, base=typ, min=element.TypeInfo.Arity.Min, max=element.TypeInfo.Arity.Max, ), ) return f.write( textwrap.dedent( """\ <xsd:complexType name="_{name}"> <xsd:sequence> <xsd:element name="item" type="{type}" minOccurs="{min}" maxOccurs="{max}" /> </xsd:sequence> </xsd:complexType> """, ).format( name=element.DottedName, type=typ, min=element.TypeInfo.Arity.Min, max=element.TypeInfo.Arity.Max or "unbounded", ), ) # ---------------------------------------------------------------------- Elements.CreateElementVisitor(OnElement).Accept( elements, include_dotted_names=include_dotted_names, ) f.write( textwrap.dedent( """\ </xsd:schema> """, ), )
def EntryPoint( delete_before=datetime.timedelta(days=7), yes=False, output_stream=sys.stdout, verbose=False, ): with StreamDecorator(output_stream).DoneManager( line_prefix='', prefix="\nResults: ", suffix='\n', ) as dm: verbose_stream = StreamDecorator(dm.stream if verbose else None, line_prefix="INFO: ") # Find the files # ---------------------------------------------------------------------- FileInfo = namedtuple( "FileInfo", [ "Name", "Type", "Fullpath", "Age", "Size", ], ) # ---------------------------------------------------------------------- t = time.time() dm.stream.write("Searching for files...") with dm.stream.DoneManager(suffix='\n', ): file_infos = [] for filename in FileSystem.WalkFiles( CurrentShell.TempDirectory, include_file_extensions=[ RepositoryBootstrapConstants.TEMPORARY_FILE_EXTENSION, ], ): name = os.path.splitext( os.path.basename(filename))[0].split('.') if len(name) == 1: type_ = '' name = name[0] else: type_ = name[-1] name = '.'.join(name[:-1]) file_infos.append( FileInfo( name, type_, filename, datetime.timedelta(seconds=t - os.stat(filename)[stat.ST_MTIME]), os.stat(filename)[stat.ST_SIZE], )) if not file_infos: dm.stream.write("No files were found.\n") return dm.result dm.stream.write("{} {} found.\n".format( inflect.no("file", len(file_infos)), inflect.plural("was", len(file_infos)), )) verbose_stream.write("\nFiles found:\n{}\n\n".format('\n'.join( [fi.Fullpath for fi in file_infos]))) # Trim the list based on age file_infos = [fi for fi in file_infos if fi.Age >= delete_before] if not file_infos: dm.stream.write( "No files were found older than {}.\n".format(delete_before)) return dm.result if not yes: total_size = 0 for fi in file_infos: total_size += fi.Size dm.stream.write( textwrap.dedent("""\ Would you like to delete these files: Name Type Size Age (days) Fullpath -------------------------- ------------------ ----------------- ------------------------------ ----------------------------------------------- {files} ? ({total_size}) [y/N] """).format( files='\n'.join([ " {name:<26} {type:18} {size:<17} {age:<30} {fullpath}" .format( name=fi.Name, type=fi.Type, size=FileSystem.GetSizeDisplay(fi.Size), age=str(fi.Age), fullpath=fi.Fullpath, ) for fi in file_infos ]), total_size=FileSystem.GetSizeDisplay(total_size), )) value = six.moves.input().strip() if not value: value = 'N' value = value.lower() if value in [ "0", "n", "no", ]: return dm.result dm.stream.write("\nRemoving files...") with dm.stream.DoneManager() as this_dm: for index, fi in enumerate(file_infos): this_dm.stream.write("Removing '{}' ({} of {})...".format( fi.Fullpath, index + 1, len(file_infos), )) with this_dm.stream.DoneManager(): FileSystem.RemoveFile(fi.Fullpath) return dm.result