class StringTypeInfo(TypeInfo): # ---------------------------------------------------------------------- # | # | Public Types # | # ---------------------------------------------------------------------- TypeName = Interface.DerivedProperty("string") CppType = Interface.DerivedProperty("std::string") CType = "char const *" # ---------------------------------------------------------------------- # | # | Public Methods # | # ---------------------------------------------------------------------- @Interface.override def GetTransformInputArgs( self, input_name="input", ): if self.IsOptional: return "Microsoft::Featurizer::Traits<nonstd::optional<std::string>>::IsNull({input_name}) ? nullptr : {input_name}->c_str()".format( input_name=input_name, ) return "{}.c_str()".format(input_name) # ---------------------------------------------------------------------- @Interface.override def GetTransformInputBufferArgs( self, input_name='input', ): raise NotImplementedError("Not implemented yet") # ---------------------------------------------------------------------- @Interface.override def GetOutputInfo( self, invocation_template, result_name="result", ): if self.IsOptional: vector_type = "nonstd::optional<std::string>" statement = "{result}_ptr ? std::string({result}_ptr) : nonstd::optional<std::string>()".format( result=result_name, ) else: vector_type = "std::string" statement = "{result}_ptr ? std::string({result}_ptr) : std::string()".format( result=result_name, ) return self.Result( vector_type, [self.Type("char const *", "{}_ptr".format(result_name))], invocation_template.format(statement), "{}_ptr".format(result_name), destroy_inline=True, )
class DatetimeTypeInfoFactory(TypeInfoFactory): # ---------------------------------------------------------------------- # | # | Public Types # | # ---------------------------------------------------------------------- TypeName = Interface.DerivedProperty("datetime") CppType = Interface.DerivedProperty( "std::chrono::system_clock::time_point") # ---------------------------------------------------------------------- # | # | Public Methods # | # ---------------------------------------------------------------------- @Interface.override def GetTransformInputArgs( self, is_input_optional, input_name="input", ): if is_input_optional: return "Microsoft::Featurizer::Traits<decltype({input_name})>::IsNull({input_name}) ? nullptr : CreateDateTimeParameter(*{input_name})".format( input_name=input_name, ) return "CreateDateTimeParameter({input_name})".format( input_name=input_name, ) # ---------------------------------------------------------------------- @Interface.override def GetOutputInfo( self, result_name="result", ): raise NotImplementedError("Not implemented yet")
class VectorTypeInfoFactory(TypeInfoFactory): # ---------------------------------------------------------------------- # | # | Public Properties # | # ---------------------------------------------------------------------- TypeName = Interface.DerivedProperty( re.compile(r"vector\<(?P<type>\S+)\>")) CSharpType = Interface.DerivedProperty("TODO1") CSharpTypeName = Interface.DerivedProperty("TODO2") # ---------------------------------------------------------------------- # | # | Public Methods # | # ---------------------------------------------------------------------- def __init__( self, custom_structs=None, custom_enums=None, member_type=None, create_type_info_factory_func=None, ): if member_type is not None: assert create_type_info_factory_func is not None match = self.TypeName.match(member_type) assert match, member_type the_type = match.group("type") type_info = create_type_info_factory_func(the_type) self._type_info = type_info # ---------------------------------------------------------------------- @Interface.override def GetNativeInputInfo(self, is_optional): return self.Result( "TODO3: Parameter decl", "TODO4: Validation statements", "TODO5: Invocation statements", "TODO6: Conversion end", "TODO7: Delete transformed data", ) # ---------------------------------------------------------------------- @Interface.override def GetNativeOutputInfo( self, is_struct=False, featurizer_name="", ): return self.Result( "TODO8: Parameter decl", "TODO9: Validation statements", "TODO10: Invocation statements", "TODO11: Conversion end", "TODO12: Delete transformed data", )
class Plugin(PluginBase): # ---------------------------------------------------------------------- # | Properties Name = Interface.DerivedProperty("Debug") Description = Interface.DerivedProperty( "Plugin that displays debug information and nothing else", ) # ---------------------------------------------------------------------- # | Methods @staticmethod @Interface.override def GetRequiredMetadataNames(): return [] # ---------------------------------------------------------------------- @staticmethod @Interface.override def GenerateOutputFilenames(context): assert "output_name" in context yield "{}.txt".format(context["output_name"]) # ---------------------------------------------------------------------- @staticmethod @Interface.override def Execute(invoke_reason, context, status_stream, verbose_stream, verbose): # <unused argument> pylint: disable = W0613 assert len( context["output_filenames"]) == 1, context["output_filenames"] output_filename = context["output_filenames"][0] status_stream.write("Writing '{}'...".format(output_filename)) with status_stream.DoneManager(): sink = six.StringIO() CommonEnvironment.Describe( context, output_stream=sink, ) sink = sink.getvalue() with open(output_filename, "w") as f: f.write( textwrap.dedent( """\ # ---------------------------------------------------------------------- # | # | Debug Output - Execute # | # ---------------------------------------------------------------------- invoke_reason: {invoke_reason} context: {context} """, ).format( invoke_reason=invoke_reason, context=StringHelpers.LeftJustify(sink.strip(), 4), ), )
class Plugin(PluginBase): # ---------------------------------------------------------------------- # | Properties Name = Interface.DerivedProperty("SharedLibraryTests") Description = Interface.DerivedProperty( "Generates code used when testing the Shared Library import/export layer", ) # ---------------------------------------------------------------------- # | Methods @staticmethod @Interface.override def Generate( open_file_func, global_custom_structs, global_custom_enums, data, output_dir, status_stream, ): result_code = 0 status_stream.write("Preprocessing data...") with status_stream.DoneManager(): type_info_data = [] for items in data: type_info_data.append([ TypeInfoData(item, global_custom_structs, global_custom_enums) for item in items ]) for desc, func in [("Generating .h files...", _GenerateHeaderFile)]: status_stream.write(desc) with status_stream.DoneManager(suffix="\n", ) as dm: for index, (items, items_type_info_data) in enumerate( zip(data, type_info_data), ): dm.stream.write( "Processing '{}' ({} of {})...".format( items[0].name, index + 1, len(data), ), ) with dm.stream.DoneManager() as this_dm: this_dm.result = func( open_file_func, output_dir, items, items_type_info_data, this_dm.stream, ) if dm.result < 0: return dm.result result_code = result_code or dm.result return result_code
class Plugin(PluginBase): # ---------------------------------------------------------------------- # | Properties Name = Interface.DerivedProperty("MLNet") Description = Interface.DerivedProperty( "Generates code used during the Shared Library import/export layer interfacing with the shared C++ functionality", ) # ---------------------------------------------------------------------- # | Methods @staticmethod @Interface.override def Generate(open_file_func, global_custom_structs, global_custom_enums, data, output_dir, status_stream): result_code = 0 status_stream.write("Preprocessing data for ML.NET...") with status_stream.DoneManager(): # Convert the types into the corresponding C#/CPP types that will be used # in the Shared Library interface. csharp_data = [] unsupported_types = set() for items in data: csharp_data.append([ _FillList(item, status_stream, unsupported_types, global_custom_structs, global_custom_enums) for item in items ], ) for desc, func in [("Generating C# files...", _GenerateCSharpFile)]: status_stream.write(desc) with status_stream.DoneManager(suffix="\n", ) as dm: for index, (items, items_csharp_data) in enumerate( zip(data, csharp_data), ): dm.stream.write( "Processing '{}' ({} of {})...".format( items[0].name, index + 1, len(data), ), ) with dm.stream.DoneManager() as this_dm: this_dm.result = func( open_file_func, output_dir, items, items_csharp_data, this_dm.stream, ) if dm.result < 0: return dm.result result_code = result_code or dm.result return result_code
class UniqueIdTypeInfo(TypeInfo): # ---------------------------------------------------------------------- # | # | Public Properties # | # ---------------------------------------------------------------------- TypeName = Interface.DerivedProperty("unique_id") CSharpType = Interface.DerivedProperty("TODO1") CSharpTypeName = Interface.DerivedProperty("TODO2") # ---------------------------------------------------------------------- # | # | Public Methods # | # ---------------------------------------------------------------------- def __init__(self, *args, member_type=None, create_type_info_func=None, **kwargs): if member_type is None: return assert create_type_info_func is not None super(UniqueIdTypeInfo, self).__init__(*args, **kwargs) if self.IsOptional: raise Exception("UniqueId types cannot be optional") # ---------------------------------------------------------------------- @Interface.override def GetNativeInputInfo(self): return self.Result( "TODO3: Parameter decl", "TODO4: Validation statements", "TODO5: Invocation statements", "TODO6: Conversion end", "TODO7: Delete transformed data", ) # ---------------------------------------------------------------------- @Interface.override def GetNativeOutputInfo( self, is_struct=False, featurizer_name="", ): return self.Result( "TODO8: Parameter decl", "TODO9: Validation statements", "TODO10: Invocation statements", "TODO11: Conversion end", "TODO12: Delete transformed data", )
class TestParser(TestParserImpl): """Parses content produced by Catch2""" # ---------------------------------------------------------------------- # | Public Properties Name = Interface.DerivedProperty("Catch2") Description = Interface.DerivedProperty("Parses Catch2 output.") # ---------------------------------------------------------------------- # | Public Methods @staticmethod @Interface.override def IsSupportedCompiler(compiler): # Many compilers are supported return True # ---------------------------------------------------------------------- @staticmethod @Interface.override def Parse(test_data): if "All tests passed" not in test_data: return -1 benchmark_data = ExtractBenchmarkOutput(test_data) if benchmark_data: return 0, benchmark_data return 0 # ---------------------------------------------------------------------- @staticmethod @Interface.override def CreateInvokeCommandLine(context, debug_on_error): if "output_filename" in context and context["output_filename"]: output_filename = context["output_filename"] elif "output_filenames" in context and context["output_filenames"]: assert len( context["output_filenames"]) == 1, context["output_filenames"] output_filename = context["output_filenames"][0] else: assert False, context dirname, basename = os.path.split(output_filename) if context.get("is_benchmark", False): flags = "[benchmark]" else: flags = "~[benchmark] --success" return 'cd "{output_dir}" && {output_name} {flags}'.format( output_dir=dirname, output_name=basename, flags=flags, )
class StringTypeInfoFactory(TypeInfoFactory): # ---------------------------------------------------------------------- # | # | Public Types # | # ---------------------------------------------------------------------- TypeName = Interface.DerivedProperty("string") CppType = Interface.DerivedProperty("std::string") # ---------------------------------------------------------------------- # | # | Public Methods # | # ---------------------------------------------------------------------- @Interface.override def GetTransformInputArgs( self, is_input_optional, input_name="input", ): if is_input_optional: return "Microsoft::Featurizer::Traits<nonstd::optional<std::string>>::IsNull({input_name}) ? nullptr : {input_name}->c_str()".format( input_name=input_name, ) return "{}.c_str()".format(input_name) # ---------------------------------------------------------------------- @Interface.override def GetOutputInfo( self, result_name="result", ): return self.Result( "std::string", textwrap.dedent( """\ char const * {result_name}_ptr(nullptr); std::size_t {result_name}_items(0); """, ).format(result_name=result_name, ), "&{result_name}_ptr, &{result_name}_items".format( result_name=result_name, ), textwrap.dedent( """\ #if (defined __apple_build_version__) results.push_back({result}_ptr ? std::string({result}_ptr) : std::string()); #else results.emplace_back({result}_ptr ? std::string({result}_ptr) : std::string()); #endif """, ).format(result=result_name, ), "{result_name}_ptr, {result_name}_items".format( result_name=result_name, ), destroy_inline=True, )
class Plugin(PluginBase): """Prints diagnostic information about the input code""" # ---------------------------------------------------------------------- # | Properties Name = Interface.DerivedProperty("Debug") Priority = Interface.DerivedProperty(1) # ---------------------------------------------------------------------- # | Methods @classmethod @Interface.override def PreprocessLines(cls, lines): return cls._Display("Pre clang-format", lines) # ---------------------------------------------------------------------- @classmethod @Interface.override def Decorate(cls, lines): return cls._Display("Post clang-format", lines) # ---------------------------------------------------------------------- @classmethod @Interface.override def PostprocessLines(cls, lines): return cls._Display("Post-Decoration", lines) # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- @staticmethod def _Display(header, lines): sys.stdout.write( textwrap.dedent( """\ # ---------------------------------------------------------------------- # | # | {} # | # ---------------------------------------------------------------------- """, ).format(header), ) for line_index, line in enumerate(lines): sys.stdout.write("{0:>3}) {1}\n".format(line_index + 1, line)) sys.stdout.write("\n") return lines
class DestinationStatementWriter(PythonDestinationStatementWriter): # ---------------------------------------------------------------------- # | Public Properties ObjectTypeDesc = Interface.DerivedProperty("a JSON object") # ---------------------------------------------------------------------- # | Methods @staticmethod @Interface.override def SerializeToString(var_name): return "_JsonToString({var_name}, pretty_print)".format( var_name=var_name, ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def GetGlobalUtilityMethods(source_writer): return textwrap.dedent( """\ # ---------------------------------------------------------------------- def _JsonToString(obj, pretty_print): if pretty_print: content = json.dumps(obj, cls=JsonEncoder, indent=2, separators=[", ", " : "]) # Remove trailing whitespace return "\\n".join([line.rstrip() for line in content.split("\\n")]) else: return json.dumps(obj, cls=JsonEncoder) """, )
class SourceStatementWriter(PythonSourceStatementWriter): # ---------------------------------------------------------------------- # | Properties ObjectTypeDesc = Interface.DerivedProperty("a YAML object") # ---------------------------------------------------------------------- # | Methods @classmethod @Interface.override def ConvenienceConversions(cls, var_name, element_or_none): content = textwrap.dedent( """\ if isinstance({var_name}, six.string_types): if FileSystem.IsFilename({var_name}): with open({var_name}) as f: {var_name} = rtyaml.load(f) else: {var_name} = rtyaml.load({var_name}) """, ).format(var_name=var_name, ) if element_or_none is not None: content += textwrap.dedent( """\ {} """, ).format( super(Plugin.SourceStatementWriter, cls).ConvenienceConversions( var_name, element_or_none)) return content
class TestExecutor(TestExecutorImpl): # ---------------------------------------------------------------------- # | Properties Name = Interface.DerivedProperty("ClangCodeCoverage") Description = Interface.DerivedProperty( "Extracts code coverage information using Clang tools.") # ---------------------------------------------------------------------- # | Methods @staticmethod @Interface.override def IsSupportedCompiler(compiler): return compiler.Name in ["CMake"] # ---------------------------------------------------------------------- _CodeCoverageExecutor = Interface.DerivedProperty( ClangCodeCoverageExecutor)
class DestinationStatementWriter(PythonDestinationStatementWriter): # ---------------------------------------------------------------------- # | Properties ObjectTypeDesc = Interface.DerivedProperty("a YAML object") # ---------------------------------------------------------------------- # | Methods @staticmethod @Interface.override def SerializeToString(var_name): return "rtyaml.dump({var_name})".format(var_name=var_name, )
class DatetimeTypeInfoFactory(TypeInfoFactory): # ---------------------------------------------------------------------- # | # | Public Properties # | # ---------------------------------------------------------------------- TypeName = Interface.DerivedProperty("datetime") CSharpType = Interface.DerivedProperty("TODO") CSharpTypeName = Interface.DerivedProperty("TODO") # ---------------------------------------------------------------------- # | # | Public Methods # | # ---------------------------------------------------------------------- @classmethod @Interface.override def GetNativeInputInfo(cls, is_optional): return cls.Result( "TODO: Parameter decl", "TODO: Validation statements or None", "TODO: invocation_statement", "TODO: conversion end", "TODO: delete transformed data statement or empty string", ) # ---------------------------------------------------------------------- @classmethod @Interface.override def GetNativeOutputInfo( cls, is_struct=False, featurizer_name="", ): return cls.Result( "TODO: Parameter decl", "TODO: Validation statements or None", "TODO: invocation_statement", "TODO: conversion end", "TODO: delete transformed data statement or empty string", )
class TestExecutorImpl(TestExecutorImplBase): # ---------------------------------------------------------------------- # | Methods @staticmethod @Interface.override def ValidateEnvironment(): if CurrentShell.CategoryName != "Windows": return "The '{}' test executor is only available on Windows".format(cls.Name) # ---------------------------------------------------------------------- @staticmethod @Interface.override def IsSupportedCompiler(compiler): return compiler.Name in ["CMake"] # ---------------------------------------------------------------------- _CodeCoverageExecutor = Interface.DerivedProperty(MSVCCodeCoverageExecutor)
class DestinationStatementWriter( PythonSerializationImpl.DestinationStatementWriter): ObjectTypeDesc = Interface.DerivedProperty( "an XML object (ElementTree)") # ---------------------------------------------------------------------- @classmethod @Interface.override def CreateCompoundElement(cls, element, attributes_var_or_none): return textwrap.dedent( """\ _CreateXmlElement( {name}, attributes={attributes}, ) """, ).format( name=cls.GetElementStatementName(element), attributes=attributes_var_or_none or "None", ) # ---------------------------------------------------------------------- @classmethod @Interface.override def CreateSimpleElement(cls, element, attributes_var_or_none, fundamental_statement): return textwrap.dedent( """\ _CreateXmlElement( {name}, attributes={attributes}, text_value={fundamental}, ) """, ).format( name=cls.GetElementStatementName(element), attributes=attributes_var_or_none or "None", fundamental=fundamental_statement, ) # ---------------------------------------------------------------------- @classmethod @Interface.override def CreateFundamentalElement(cls, element, fundamental_statement): return textwrap.dedent( """\ _CreateXmlElement( {name}, text_value={statement}, ) """, ).format( name=cls.GetElementStatementName(element), statement=fundamental_statement, ) # ---------------------------------------------------------------------- @classmethod @Interface.override def CreateCollection(cls, element, result_name): return "cls._CreateXmlCollection({}, {})".format( cls.GetElementStatementName(element), result_name) # ---------------------------------------------------------------------- @classmethod @Interface.override def AppendChild(cls, child_element, parent_var_name, var_name_or_none): if getattr(child_element, "IsAttribute", False): return "{parent_var_name}.attrib[{element_name}] = {var_name}".format( parent_var_name=parent_var_name, element_name=cls.GetElementStatementName(child_element), var_name=var_name_or_none, ) if var_name_or_none is None: var_name_or_none = "_CreateXmlElement({})".format( cls.GetElementStatementName(child_element)) return "{parent_var_name}.append({var_name})".format( parent_var_name=parent_var_name, var_name=var_name_or_none, ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def SerializeToString(var_name): return textwrap.dedent( """\ ET.tostring( _XmlPrettyPrint({var_name}) if pretty_print else {var_name}, encoding="utf-8", method="xml", ).decode("utf-8") """, ).format(var_name=var_name, ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def GetClassUtilityMethods(source_writer): return textwrap.dedent( """\ # ---------------------------------------------------------------------- @staticmethod def _CreateXmlCollection(element_name, items_or_none): result = _CreateXmlElement(element_name) for item in (items_or_none or []): item.tag = "{collection_item_name}" result.append(item) return result """, ).format( collection_item_name=Plugin.COLLECTION_ITEM_NAME, ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def GetGlobalUtilityMethods(source_writer): return textwrap.dedent( """\ # ---------------------------------------------------------------------- def _CreateXmlElement( element_name, attributes=None, text_value=None, ): result = ET.Element( element_name, attrib=attributes or {}, ) if text_value is not None: result.text = text_value return result # ---------------------------------------------------------------------- def _XmlPrettyPrint(elem, level=0): original = elem i = "\\n" + level * " " if elem: if not elem.text or not elem.text.strip(): elem.text = i + " " if not elem.tail or not elem.tail.strip(): elem.tail = i for child in elem: _XmlPrettyPrint(child, level + 1) # <Using possibly undefined loop variable 'child'> pylint: disable = W0631 if not child.tail or not child.tail.strip(): child.tail = i else: if level and (not elem.tail or not elem.tail.strip()): elem.tail = i return original """, )
class UInt64TypeInfoFactory(_ScalarTypeInfoFactory): TypeName = Interface.DerivedProperty("uint64") CppType = Interface.DerivedProperty("std::uint64_t") CType = Interface.DerivedProperty("uint64_t")
class CodeGenerator( AtomicInputProcessingMixin, ConditionalInvocationQueryMixin, MultipleOutputMixin, CodeGeneratorBase, ): # ---------------------------------------------------------------------- # | Types ContextCode = namedtuple("ContextCode", ["filename", "var_name"]) # ---------------------------------------------------------------------- # | Properties Name = Interface.DerivedProperty("Jinja2CodeGenerator") Description = Interface.DerivedProperty( "Processes a Jinja2 template and produces output") InputTypeInfo = Interface.DerivedProperty( CommandLine.FilenameTypeInfo( validation_expression=".+?\.jinja2(?:\..+)?", ), ) # ---------------------------------------------------------------------- # | Methods # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- @classmethod @Interface.override def _GetOptionalMetadata(cls): return [("jinja2_context", {}), ("jinja2_context_code", []), ("preserve_dir_structure", False), ("ignore_errors", False), ("debug", False)] + super( CodeGenerator, cls, )._GetOptionalMetadata() # ---------------------------------------------------------------------- @classmethod @Interface.override def _GetRequiredMetadataNames(cls): return ["output_dir"] + super(CodeGenerator, cls)._GetRequiredMetadataNames() # ---------------------------------------------------------------------- @classmethod @Interface.override def _CreateContext(cls, metadata, status_stream): jinja2_context = {} # Load the custom context defined in code for context_code in metadata["jinja2_context_code"]: dirname, basename = os.path.split(context_code) basename = os.path.splitext(basename)[0] sys.path.insert(0, dirname) with CallOnExit(lambda: sys.path.pop(0)): mod = importlib.import_module(basename) var = getattr(mod, context_code.var_name) del mod if isinstance(var, dict): for k, v in six.iteritems(var): jinja2_context[k] = v else: jinja2_context[context_code.var_name] = var del metadata["jinja2_context_code"] # Load the custom context for k, v in six.iteritems(metadata["jinja2_context"]): if len(v) == 1: jinja2_context[k] = v[0] else: jinja2_context[k] = v metadata["jinja2_context"] = jinja2_context # Calculate the hashes of the input filenames. We will use this information # during comparison to determine if an input file has changed. It appears # that this value isn't used, but it is actually used when comparing the # context of two different invocations. # ---------------------------------------------------------------------- def CalculateHash(input_filename): with open(input_filename, "rb") as f: return hashlib.sha256(f.read()).digest() # ---------------------------------------------------------------------- metadata["hashes"] = [ CalculateHash(input_filename) for input_filename in metadata["inputs"] ] # Get the output filenames if not metadata["preserve_dir_structure"]: # ---------------------------------------------------------------------- def GetBaseDir(input_filename): return '' # ---------------------------------------------------------------------- else: if len(metadata["inputs"]) == 1: common_prefix = os.path.dirname(metadata["inputs"][0]) else: common_prefix = FileSystem.GetCommonPath(*metadata["inputs"]) # ---------------------------------------------------------------------- def GetBaseDir(input_filename): return FileSystem.TrimPath(input_filename, common_prefix) # ---------------------------------------------------------------------- output_filenames = [] for input_filename in metadata["inputs"]: output_filenames.append( os.path.join( metadata["output_dir"], GetBaseDir(input_filename), '.'.join([ part for part in os.path.basename(input_filename).split(".") if part != "jinja2" ]), ), ) metadata["output_filenames"] = output_filenames return super(CodeGenerator, cls)._CreateContext(metadata, status_stream) # ---------------------------------------------------------------------- @classmethod @Interface.override def _InvokeImpl(cls, invoke_reason, context, status_stream, verbose_stream, verbose): # ---------------------------------------------------------------------- class RelativeFileSystemLoader(FileSystemLoader): # ---------------------------------------------------------------------- def __init__( self, input_filename, searchpath=None, *args, **kwargs, ): super(RelativeFileSystemLoader, self).__init__( searchpath=[os.path.dirname(input_filename)] + (searchpath or []), *args, **kwargs) # ---------------------------------------------------------------------- def get_source(self, environment, template): method = super(RelativeFileSystemLoader, self).get_source try: return method(environment, template) except exceptions.TemplateNotFound: for searchpath in reversed(self.searchpath): potential_template = os.path.normpath( os.path.join(searchpath, template).replace('/', os.path.sep)) if os.path.isfile(potential_template): dirname, basename = os.path.split( potential_template) self.searchpath.append(dirname) return method(environment, template) raise # ---------------------------------------------------------------------- with status_stream.DoneManager(display=False, ) as dm: for index, (input_filename, output_filename) in enumerate( zip( context["inputs"], context["output_filenames"], )): status_stream.write( "Processing '{}' ({} of {})...".format( input_filename, index + 1, len(context["inputs"]), ), ) with dm.stream.DoneManager( suppress_exceptions=True, ) as this_dm: try: # ---------------------------------------------------------------------- def ReadFileFilter(value): potential_filename = os.path.join( os.path.dirname(input_filename), value) if not os.path.isfile(potential_filename): return "<< '{}' was not found >>".format( potential_filename) with open(potential_filename) as f: return f.read() # ---------------------------------------------------------------------- loader = RelativeFileSystemLoader(input_filename) if context["debug"]: from jinja2 import meta env = Environment(loader=loader, ) with open(input_filename) as f: content = env.parse(f.read()) this_dm.stream.write("Variables:\n{}\n".format( "\n".join([ " - {}".format(var) for var in meta.find_undeclared_variables(content) ]))) continue if context["ignore_errors"]: undef = Undefined else: undef = StrictUndefined env = Environment( trim_blocks=True, lstrip_blocks=True, loader=loader, undefined=undef, ) env.tests["valid_file"] = lambda value: os.path.isfile( os.path.dirname(input_filename), value) env.filters[ "doubleslash"] = lambda value: value.replace( "\\", "\\\\") # Technically speaking, this isn't required as Jinja's import/include/extend functionality # superseeds this functionality. However, it remains in the name of backwards compatibility. env.filters["read_file"] = ReadFileFilter with open(input_filename) as f: template = env.from_string(f.read()) try: content = template.render( **context["jinja2_context"]) except exceptions.UndefinedError as ex: this_dm.stream.write("ERROR: {}\n".format(str(ex))) this_dm.result = -1 continue with open(output_filename, "w") as f: f.write(content) except: this_dm.result = -1 raise return dm.result
class Plugin(PythonSerializationImpl): # ---------------------------------------------------------------------- # | Properties Name = Interface.DerivedProperty("PythonYaml") Description = Interface.DerivedProperty( "Creates python code that is able to serialize and deserialize python objects to YAML" ) # ---------------------------------------------------------------------- # | Methods @classmethod @Interface.override def GetAdditionalGeneratorItems(cls, context): return [ _script_fullpath, PythonDestinationStatementWriter, PythonSourceStatementWriter ] + super(Plugin, cls).GetAdditionalGeneratorItems(context) # ---------------------------------------------------------------------- # | Private Types @Interface.staticderived class SourceStatementWriter(PythonSourceStatementWriter): # ---------------------------------------------------------------------- # | Properties ObjectTypeDesc = Interface.DerivedProperty("a YAML object") # ---------------------------------------------------------------------- # | Methods @classmethod @Interface.override def ConvenienceConversions(cls, var_name, element_or_none): content = textwrap.dedent( """\ if isinstance({var_name}, six.string_types): if FileSystem.IsFilename({var_name}): with open({var_name}) as f: {var_name} = rtyaml.load(f) else: {var_name} = rtyaml.load({var_name}) """, ).format(var_name=var_name, ) if element_or_none is not None: content += textwrap.dedent( """\ {} """, ).format( super(Plugin.SourceStatementWriter, cls).ConvenienceConversions( var_name, element_or_none)) return content # ---------------------------------------------------------------------- @Interface.staticderived class DestinationStatementWriter(PythonDestinationStatementWriter): # ---------------------------------------------------------------------- # | Properties ObjectTypeDesc = Interface.DerivedProperty("a YAML object") # ---------------------------------------------------------------------- # | Methods @staticmethod @Interface.override def SerializeToString(var_name): return "rtyaml.dump({var_name})".format(var_name=var_name, ) # ---------------------------------------------------------------------- # | Private Properties _SupportAttributes = Interface.DerivedProperty(False) _SupportAnyElements = Interface.DerivedProperty(True) _SupportDictionaryElements = Interface.DerivedProperty(True) _TypeInfoSerializationName = Interface.DerivedProperty("YamlSerialization") _SourceStatementWriter = Interface.DerivedProperty(SourceStatementWriter) _DestinationStatementWriter = Interface.DerivedProperty( DestinationStatementWriter) # ---------------------------------------------------------------------- # | Private Methods @staticmethod @Interface.override def _WriteFileHeader(output_stream): output_stream.write( textwrap.dedent( """\ import rtyaml from CommonEnvironment.CallOnExit import CallOnExit from CommonEnvironment import FileSystem from CommonEnvironment.TypeInfo.FundamentalTypes.Serialization.YamlSerialization import YamlSerialization """, ), ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def _WriteFileFooter(output_stream): output_stream.write( textwrap.dedent( """\ # ---------------------------------------------------------------------- def _ObjectToYaml(dumper, data): d = dict(data.__dict__) for k in list(six.iterkeys(d)): if k.startswith("_"): del d[k] return dumper.represent_dict(d) rtyaml.Dumper.add_representer(Object, _ObjectToYaml) """, ), )
class FloatTypeInfoFactory(_ScalarTypeInfoFactory): TypeName = Interface.DerivedProperty("float") CppType = Interface.DerivedProperty("std::float_t") CType = Interface.DerivedProperty("float")
class PythonSourceStatementWriter(SourceStatementWriter): ObjectTypeDesc = Interface.DerivedProperty("a python object") # ---------------------------------------------------------------------- @classmethod @Interface.override def ConvenienceConversions(cls, var_name, element_or_none): if element_or_none is None: return "" return textwrap.dedent( """\ if not isinstance({var_name}, list): if isinstance({var_name}, dict) and "{name}" in {var_name}: {var_name} = {var_name}["{name}"] elif not isinstance({var_name}, dict) and hasattr({var_name}, "{name}"): {var_name} = getattr({var_name}, "{name}") elif is_root: {var_name} = DoesNotExist """, ).format( var_name=var_name, name=element_or_none.Name, ) # ---------------------------------------------------------------------- @classmethod @Interface.override def GetChild( cls, var_name, child_element, is_simple_schema_fundamental=False, ): if is_simple_schema_fundamental: is_optional = False else: is_optional = child_element.TypeInfo.Arity.Min == 0 return textwrap.dedent( """\ cls._GetPythonAttribute( {var_name}, {name}, is_optional={is_optional}, ) """, ).format( var_name=var_name, name=cls.GetElementStatementName(child_element), is_optional=is_optional, ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def GetFundamental(var_name, child_element): return var_name # ---------------------------------------------------------------------- @staticmethod @Interface.override def GetAdditionalDataChildren(): return '[(k, v) for k, v in six.iteritems(source if isinstance(source, dict) else getattr(source, "__dict__", {})) if not k.startswith("_") and k not in exclude_names]' # ---------------------------------------------------------------------- @classmethod @Interface.override def CreateAdditionalDataItem(cls, dest_writer, name_var_name, source_var_name): temporary_element = cls.CreateTemporaryElement(name_var_name, "1") temporary_children_element = cls.CreateTemporaryElement("k", "+") return textwrap.dedent( """\ # The following types should be returned directly without additional conversion if isinstance({source_var_name}, (int, float, str, bool)): return {source_var_name} assert not isinstance({source_var_name}, list), {source_var_name} if not isinstance({source_var_name}, dict): {source_var_name} = {source_var_name}.__dict__ source_attribute_names = {source_var_name}.get("{attribute_names}", set()) attributes = OrderedDict() items = OrderedDict() for k, v in six.iteritems(source): if k.startswith("_"): continue if k in source_attribute_names: attributes[k] = v else: items[k] = v if len(items) == 1 and next(six.iterkeys(items)) == {source_var_name}.get("{fundamental_name}", None): return {simple_statement} result = {compound_statement} for k, v in six.iteritems(items): try: if isinstance(v, list): new_items = [] for index, child in enumerate(v): try: new_items.append(cls._CreateAdditionalDataItem("item", child)) except: _DecorateActiveException("Index {{}}".format(index)) {append_children} else: new_item = cls._CreateAdditionalDataItem(k, v) {append_child} except: _DecorateActiveException(k) return result """, ).format( source_var_name=source_var_name, attribute_names=cls.ATTRIBUTES_ATTRIBUTE_NAME, fundamental_name=cls.SIMPLE_ELEMENT_FUNDAMENTAL_ATTRIBUTE_NAME, simple_statement=StringHelpers.LeftJustify( dest_writer.CreateSimpleElement( temporary_element, "attributes", '{}[{}["{}"]]'.format(source_var_name, source_var_name, cls.SIMPLE_ELEMENT_FUNDAMENTAL_ATTRIBUTE_NAME), ), 4, ).strip(), compound_statement=dest_writer.CreateCompoundElement(temporary_element, "attributes").strip(), append_children=StringHelpers.LeftJustify( dest_writer.AppendChild(temporary_children_element, "result", dest_writer.CreateCollection(temporary_children_element, "new_items")), 12, ).strip(), append_child=StringHelpers.LeftJustify(dest_writer.AppendChild(cls.CreateTemporaryElement("k", "1"), "result", "new_item"), 8).strip(), ) # ---------------------------------------------------------------------- @classmethod @Interface.override def GetGlobalUtilityMethods(cls, dest_writer): return PythonStatementWriterMixin.GetGlobalUtilityMethods(cls.ATTRIBUTES_ATTRIBUTE_NAME) # ---------------------------------------------------------------------- @classmethod @Interface.override def GetClassUtilityMethods(cls, dest_writer): return textwrap.dedent( """\ # ---------------------------------------------------------------------- @staticmethod def _GetPythonAttribute( item, attribute_name, is_optional=False, ): if not isinstance(item, dict): if hasattr(item, "__dict__"): item = item.__dict__ else: item = {} value = item.get(attribute_name, DoesNotExist) if value is DoesNotExist and not is_optional: raise SerializeException("No items were found") return value """, )
class DoubleTypeInfoFactory(_ScalarTypeInfoFactory): TypeName = Interface.DerivedProperty("double") CppType = Interface.DerivedProperty("std::double_t") CType = Interface.DerivedProperty("double")
class Plugin(PythonSerializationImpl): # ---------------------------------------------------------------------- # | Properties Name = Interface.DerivedProperty("PythonJson") Description = Interface.DerivedProperty( "Creates python code that is able to serialize and deserialize python objects to JSON" ) # ---------------------------------------------------------------------- # | Methods @classmethod @Interface.override def GetAdditionalGeneratorItems(cls, context): return [ _script_fullpath, PythonDestinationStatementWriter, PythonSourceStatementWriter ] + super(Plugin, cls).GetAdditionalGeneratorItems(context) # ---------------------------------------------------------------------- # | Private Types @Interface.staticderived class SourceStatementWriter(PythonSourceStatementWriter): # ---------------------------------------------------------------------- # | Public Properties ObjectTypeDesc = Interface.DerivedProperty("a JSON object") # ---------------------------------------------------------------------- # | Methods @classmethod @Interface.override def ConvenienceConversions(cls, var_name, element_or_none): content = textwrap.dedent( """\ if isinstance({var_name}, six.string_types): if FileSystem.IsFilename({var_name}): with open({var_name}) as f: {var_name} = json.load(f) else: {var_name} = json.loads({var_name}) """, ).format(var_name=var_name, ) if element_or_none is not None: content += textwrap.dedent( """\ {} """, ).format( super(Plugin.SourceStatementWriter, cls).ConvenienceConversions( var_name, element_or_none)) return content # ---------------------------------------------------------------------- @Interface.staticderived class DestinationStatementWriter(PythonDestinationStatementWriter): # ---------------------------------------------------------------------- # | Public Properties ObjectTypeDesc = Interface.DerivedProperty("a JSON object") # ---------------------------------------------------------------------- # | Methods @staticmethod @Interface.override def SerializeToString(var_name): return "_JsonToString({var_name}, pretty_print)".format( var_name=var_name, ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def GetGlobalUtilityMethods(source_writer): return textwrap.dedent( """\ # ---------------------------------------------------------------------- def _JsonToString(obj, pretty_print): if pretty_print: content = json.dumps(obj, cls=JsonEncoder, indent=2, separators=[", ", " : "]) # Remove trailing whitespace return "\\n".join([line.rstrip() for line in content.split("\\n")]) else: return json.dumps(obj, cls=JsonEncoder) """, ) # ---------------------------------------------------------------------- # | Private Properties _SupportAttributes = Interface.DerivedProperty(False) _SupportAnyElements = Interface.DerivedProperty(True) _SupportDictionaryElements = Interface.DerivedProperty(True) _TypeInfoSerializationName = Interface.DerivedProperty("JsonSerialization") _SourceStatementWriter = Interface.DerivedProperty(SourceStatementWriter) _DestinationStatementWriter = Interface.DerivedProperty( DestinationStatementWriter) # ---------------------------------------------------------------------- # | Private Methods @staticmethod @Interface.override def _WriteFileHeader(output_stream): output_stream.write( textwrap.dedent( """\ import json from CommonEnvironment import FileSystem from CommonEnvironment.TypeInfo.FundamentalTypes.Serialization.JsonSerialization import JsonSerialization # ---------------------------------------------------------------------- class JsonEncoder(json.JSONEncoder): def default(self, o): if isinstance(o, Object): d = copy.deepcopy(o.__dict__) for k in list(six.iterkeys(d)): if k.startswith("_"): del d[k] return d return getattr(o, "__dict__", o) """, ), ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def _WriteFileFooter(output_stream): # Nothing to do here pass
class Plugin(PythonSerializationImpl): COLLECTION_ITEM_NAME = "item" # ---------------------------------------------------------------------- # | Properties Name = Interface.DerivedProperty("PythonXml") Description = Interface.DerivedProperty( "Creates Python code that is able to serialize and deserialize python objects to XML" ) # ---------------------------------------------------------------------- # | Methods @classmethod @Interface.override def GetAdditionalGeneratorItems(cls, context): return [_script_fullpath] + super( Plugin, cls).GetAdditionalGeneratorItems(context) # ---------------------------------------------------------------------- # | Private Types @Interface.staticderived class SourceStatementWriter(PythonSerializationImpl.SourceStatementWriter): ObjectTypeDesc = Interface.DerivedProperty( "an XML object (ElementTree)") # ---------------------------------------------------------------------- @staticmethod @Interface.override def ConvenienceConversions(var_name, element_or_none): content = textwrap.dedent( """\ if isinstance({var_name}, six.string_types): if FileSystem.IsFilename({var_name}): with open({var_name}) as f: {var_name} = f.read() {var_name} = ET.fromstring({var_name}) """, ).format(var_name=var_name, ) if element_or_none is not None: content += textwrap.dedent( """\ potential_child = _GetXmlElement( {var_name}, "{name}", is_optional=True, is_collection={is_collection}, ) if potential_child is not DoesNotExist or is_root: {var_name} = potential_child """, ).format( var_name=var_name, name=element_or_none.Name, is_optional=element_or_none.TypeInfo.Arity.Min == 0, is_collection=element_or_none.TypeInfo.Arity.IsCollection, ) return content # ---------------------------------------------------------------------- @classmethod @Interface.override def GetChild( cls, var_name, child_element, is_simple_schema_fundamental=False, ): if is_simple_schema_fundamental: return cls.GetFundamental(var_name, child_element) if getattr(child_element, "IsAttribute", False): return textwrap.dedent( """\ cls._GetXmlAttribute( {var_name}, {name}, is_optional={is_optional}, ) """, ).format( var_name=var_name, name=cls.GetElementStatementName(child_element), is_optional=child_element.TypeInfo.Arity.Min == 0, ) return textwrap.dedent( """\ _GetXmlElement( {var_name}, {name}, is_optional={is_optional}, is_collection={is_collection}, ) """, ).format( var_name=var_name, name=cls.GetElementStatementName(child_element), is_optional=child_element.TypeInfo.Arity.Min == 0, is_collection=child_element.TypeInfo.Arity.IsCollection, ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def GetFundamental(var_name, child_element): if getattr(child_element, "IsAttribute", False): return var_name return "cls.GetText({})".format(var_name) # ---------------------------------------------------------------------- @classmethod @Interface.override def GetAdditionalDataChildren(cls): return "cls._GenerateAdditionalDataChildren(source, exclude_names)" # ---------------------------------------------------------------------- @classmethod @Interface.override def CreateAdditionalDataItem(cls, dest_writer, name_var_name, source_var_name): temporary_element = cls.CreateTemporaryElement( "{}.tag".format(source_var_name), "1") return textwrap.dedent( """\ attributes = OrderedDict() for k, v in six.iteritems({source_var_name}.attrib): if k.startswith("_"): continue attributes[k] = v if {source_var_name}.text and {source_var_name}.text.strip() and not {source_var_name}: return {simple_element} result = {compound_statement} for child_name, child_or_children in cls._GenerateAdditionalDataChildren({source_var_name}, set()): try: if isinstance(child_or_children, list): new_items = [] for index, child in enumerate(child_or_children): try: new_items.append(cls._CreateAdditionalDataItem("{collection_item_name}", child)) except: _DecorateActiveException("Index {{}}".format(index)) if child_name is None: result = new_items break {append_children} else: assert child_name is not None new_item = cls._CreateAdditionalDataItem(child_name, child_or_children) {append_child} except: _DecorateActiveException(child_name) return result """, ).format( source_var_name=source_var_name, collection_item_name=Plugin.COLLECTION_ITEM_NAME, compound_statement=dest_writer.CreateCompoundElement( temporary_element, "attributes").strip(), simple_element=StringHelpers.LeftJustify( dest_writer.CreateSimpleElement( temporary_element, "attributes", "{}.text".format(source_var_name)), 4).strip(), append_children=StringHelpers.LeftJustify( dest_writer.AppendChild( cls.CreateTemporaryElement("child_name", "+"), "result", "new_items"), 12).strip(), append_child=StringHelpers.LeftJustify( dest_writer.AppendChild( cls.CreateTemporaryElement("child_name", "1"), "result", "new_item"), 8).strip(), ) # ---------------------------------------------------------------------- @classmethod @Interface.override def GetClassUtilityMethods(cls, dest_writer): return textwrap.dedent( """\ # ---------------------------------------------------------------------- @staticmethod def _GetXmlAttribute( element, attribute_name, is_optional=False, ): value = element.attrib.get(attribute_name, DoesNotExist) if value is DoesNotExist and not is_optional: raise SerializeException("The attribute '{{}}' does not exist".format(attribute_name)) return value # ---------------------------------------------------------------------- @staticmethod def GetText(item): if item is None: item = "" if hasattr(item, "text"): item = item.text return item.strip() # ---------------------------------------------------------------------- @staticmethod def _GenerateAdditionalDataChildren(element, exclude_names): children = OrderedDict() for child in element: if child.tag.startswith("_") or child.tag in exclude_names: continue children.setdefault(child.tag, []).append(child) if len(children) == 1 and next(six.iterkeys(children)) == "{collection_item_name}": value = next(six.itervalues(children)) if not isinstance(value, list): value = [value] yield None, value else: for k, v in six.iteritems(children): if len(v) == 1: yield k, v[0] else: yield k, v """, ).format( collection_item_name=Plugin.COLLECTION_ITEM_NAME, ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def GetGlobalUtilityMethods(dest_writer): return textwrap.dedent( """\ # ---------------------------------------------------------------------- def _GetXmlElement( element, child_name, is_optional=False, is_collection=False, ): children = element.findall(child_name) if not children: if is_optional: return DoesNotExist raise SerializeException("No elements were found") if len(children) != 1: raise SerializeException("Multiple items were found ({{}})".format(len(children))) result = children[0] if is_collection: result = result.findall("{collection_item_name}") return result """, ).format( collection_item_name=Plugin.COLLECTION_ITEM_NAME, ) # ---------------------------------------------------------------------- @Interface.staticderived class DestinationStatementWriter( PythonSerializationImpl.DestinationStatementWriter): ObjectTypeDesc = Interface.DerivedProperty( "an XML object (ElementTree)") # ---------------------------------------------------------------------- @classmethod @Interface.override def CreateCompoundElement(cls, element, attributes_var_or_none): return textwrap.dedent( """\ _CreateXmlElement( {name}, attributes={attributes}, ) """, ).format( name=cls.GetElementStatementName(element), attributes=attributes_var_or_none or "None", ) # ---------------------------------------------------------------------- @classmethod @Interface.override def CreateSimpleElement(cls, element, attributes_var_or_none, fundamental_statement): return textwrap.dedent( """\ _CreateXmlElement( {name}, attributes={attributes}, text_value={fundamental}, ) """, ).format( name=cls.GetElementStatementName(element), attributes=attributes_var_or_none or "None", fundamental=fundamental_statement, ) # ---------------------------------------------------------------------- @classmethod @Interface.override def CreateFundamentalElement(cls, element, fundamental_statement): return textwrap.dedent( """\ _CreateXmlElement( {name}, text_value={statement}, ) """, ).format( name=cls.GetElementStatementName(element), statement=fundamental_statement, ) # ---------------------------------------------------------------------- @classmethod @Interface.override def CreateCollection(cls, element, result_name): return "cls._CreateXmlCollection({}, {})".format( cls.GetElementStatementName(element), result_name) # ---------------------------------------------------------------------- @classmethod @Interface.override def AppendChild(cls, child_element, parent_var_name, var_name_or_none): if getattr(child_element, "IsAttribute", False): return "{parent_var_name}.attrib[{element_name}] = {var_name}".format( parent_var_name=parent_var_name, element_name=cls.GetElementStatementName(child_element), var_name=var_name_or_none, ) if var_name_or_none is None: var_name_or_none = "_CreateXmlElement({})".format( cls.GetElementStatementName(child_element)) return "{parent_var_name}.append({var_name})".format( parent_var_name=parent_var_name, var_name=var_name_or_none, ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def SerializeToString(var_name): return textwrap.dedent( """\ ET.tostring( _XmlPrettyPrint({var_name}) if pretty_print else {var_name}, encoding="utf-8", method="xml", ).decode("utf-8") """, ).format(var_name=var_name, ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def GetClassUtilityMethods(source_writer): return textwrap.dedent( """\ # ---------------------------------------------------------------------- @staticmethod def _CreateXmlCollection(element_name, items_or_none): result = _CreateXmlElement(element_name) for item in (items_or_none or []): item.tag = "{collection_item_name}" result.append(item) return result """, ).format( collection_item_name=Plugin.COLLECTION_ITEM_NAME, ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def GetGlobalUtilityMethods(source_writer): return textwrap.dedent( """\ # ---------------------------------------------------------------------- def _CreateXmlElement( element_name, attributes=None, text_value=None, ): result = ET.Element( element_name, attrib=attributes or {}, ) if text_value is not None: result.text = text_value return result # ---------------------------------------------------------------------- def _XmlPrettyPrint(elem, level=0): original = elem i = "\\n" + level * " " if elem: if not elem.text or not elem.text.strip(): elem.text = i + " " if not elem.tail or not elem.tail.strip(): elem.tail = i for child in elem: _XmlPrettyPrint(child, level + 1) # <Using possibly undefined loop variable 'child'> pylint: disable = W0631 if not child.tail or not child.tail.strip(): child.tail = i else: if level and (not elem.tail or not elem.tail.strip()): elem.tail = i return original """, ) # ---------------------------------------------------------------------- # | Private Properties _SupportAttributes = Interface.DerivedProperty(True) _SupportAnyElements = Interface.DerivedProperty(True) _SupportDictionaryElements = Interface.DerivedProperty(False) _TypeInfoSerializationName = Interface.DerivedProperty("XmlSerialization") _SourceStatementWriter = Interface.DerivedProperty(SourceStatementWriter) _DestinationStatementWriter = Interface.DerivedProperty( DestinationStatementWriter) # ---------------------------------------------------------------------- # | Private Methods @staticmethod @Interface.override def _WriteFileHeader(output_stream): output_stream.write( textwrap.dedent( """\ import xml.etree.ElementTree as ET from CommonEnvironment import FileSystem from CommonEnvironment.TypeInfo.FundamentalTypes.Serialization.XmlSerialization import XmlSerialization """, ), ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def _WriteFileFooter(output_stream): # Nothing to do here pass
class Plugin(RelationalPluginImpl): # ---------------------------------------------------------------------- # | # | Public Properties # | # ---------------------------------------------------------------------- Name = Interface.DerivedProperty("DebugRelational") Description = Interface.DerivedProperty( "Displays information associated with objects created by the base Relational plugin class" ) # ---------------------------------------------------------------------- # | # | Public Methods # | # ---------------------------------------------------------------------- @classmethod @Interface.override def GetAdditionalGeneratorItems(cls, context): return super(Plugin, cls).GetAdditionalGeneratorItems(context) + [ RelationalPluginImpl ] # ---------------------------------------------------------------------- @classmethod @Interface.override def GenerateOutputFilenames( cls, context, all_objects=None, ): return [ os.path.join(context["output_dir"], "{}.txt".format(context["output_name"])) ] # ---------------------------------------------------------------------- @classmethod @Interface.override def Generate( cls, simple_schema_generator, invoke_reason, input_filenames, output_filenames, name, elements, include_indexes, status_stream, verbose_stream, verbose, ): with open(output_filenames[0], "w") as f: f.write(cls._GenerateFileHeader()) for obj in cls.AllObjects: f.write( textwrap.dedent( """\ # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- # ---------------------------------------------------------------------- Unique Name: {unique} Singular Name: {singular} Plural Name: {plural} """, ).format( unique=obj.UniqueName, singular=obj.SingularName, plural=obj.PluralName, ), ) for child in obj.children: f.write( textwrap.dedent( """\ # ---------------------------------------------------------------------- {} {} """, ).format( child.Name, child.Item, ), ) f.write("\n")
class SourceStatementWriter(PythonSerializationImpl.SourceStatementWriter): ObjectTypeDesc = Interface.DerivedProperty( "an XML object (ElementTree)") # ---------------------------------------------------------------------- @staticmethod @Interface.override def ConvenienceConversions(var_name, element_or_none): content = textwrap.dedent( """\ if isinstance({var_name}, six.string_types): if FileSystem.IsFilename({var_name}): with open({var_name}) as f: {var_name} = f.read() {var_name} = ET.fromstring({var_name}) """, ).format(var_name=var_name, ) if element_or_none is not None: content += textwrap.dedent( """\ potential_child = _GetXmlElement( {var_name}, "{name}", is_optional=True, is_collection={is_collection}, ) if potential_child is not DoesNotExist or is_root: {var_name} = potential_child """, ).format( var_name=var_name, name=element_or_none.Name, is_optional=element_or_none.TypeInfo.Arity.Min == 0, is_collection=element_or_none.TypeInfo.Arity.IsCollection, ) return content # ---------------------------------------------------------------------- @classmethod @Interface.override def GetChild( cls, var_name, child_element, is_simple_schema_fundamental=False, ): if is_simple_schema_fundamental: return cls.GetFundamental(var_name, child_element) if getattr(child_element, "IsAttribute", False): return textwrap.dedent( """\ cls._GetXmlAttribute( {var_name}, {name}, is_optional={is_optional}, ) """, ).format( var_name=var_name, name=cls.GetElementStatementName(child_element), is_optional=child_element.TypeInfo.Arity.Min == 0, ) return textwrap.dedent( """\ _GetXmlElement( {var_name}, {name}, is_optional={is_optional}, is_collection={is_collection}, ) """, ).format( var_name=var_name, name=cls.GetElementStatementName(child_element), is_optional=child_element.TypeInfo.Arity.Min == 0, is_collection=child_element.TypeInfo.Arity.IsCollection, ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def GetFundamental(var_name, child_element): if getattr(child_element, "IsAttribute", False): return var_name return "cls.GetText({})".format(var_name) # ---------------------------------------------------------------------- @classmethod @Interface.override def GetAdditionalDataChildren(cls): return "cls._GenerateAdditionalDataChildren(source, exclude_names)" # ---------------------------------------------------------------------- @classmethod @Interface.override def CreateAdditionalDataItem(cls, dest_writer, name_var_name, source_var_name): temporary_element = cls.CreateTemporaryElement( "{}.tag".format(source_var_name), "1") return textwrap.dedent( """\ attributes = OrderedDict() for k, v in six.iteritems({source_var_name}.attrib): if k.startswith("_"): continue attributes[k] = v if {source_var_name}.text and {source_var_name}.text.strip() and not {source_var_name}: return {simple_element} result = {compound_statement} for child_name, child_or_children in cls._GenerateAdditionalDataChildren({source_var_name}, set()): try: if isinstance(child_or_children, list): new_items = [] for index, child in enumerate(child_or_children): try: new_items.append(cls._CreateAdditionalDataItem("{collection_item_name}", child)) except: _DecorateActiveException("Index {{}}".format(index)) if child_name is None: result = new_items break {append_children} else: assert child_name is not None new_item = cls._CreateAdditionalDataItem(child_name, child_or_children) {append_child} except: _DecorateActiveException(child_name) return result """, ).format( source_var_name=source_var_name, collection_item_name=Plugin.COLLECTION_ITEM_NAME, compound_statement=dest_writer.CreateCompoundElement( temporary_element, "attributes").strip(), simple_element=StringHelpers.LeftJustify( dest_writer.CreateSimpleElement( temporary_element, "attributes", "{}.text".format(source_var_name)), 4).strip(), append_children=StringHelpers.LeftJustify( dest_writer.AppendChild( cls.CreateTemporaryElement("child_name", "+"), "result", "new_items"), 12).strip(), append_child=StringHelpers.LeftJustify( dest_writer.AppendChild( cls.CreateTemporaryElement("child_name", "1"), "result", "new_item"), 8).strip(), ) # ---------------------------------------------------------------------- @classmethod @Interface.override def GetClassUtilityMethods(cls, dest_writer): return textwrap.dedent( """\ # ---------------------------------------------------------------------- @staticmethod def _GetXmlAttribute( element, attribute_name, is_optional=False, ): value = element.attrib.get(attribute_name, DoesNotExist) if value is DoesNotExist and not is_optional: raise SerializeException("The attribute '{{}}' does not exist".format(attribute_name)) return value # ---------------------------------------------------------------------- @staticmethod def GetText(item): if item is None: item = "" if hasattr(item, "text"): item = item.text return item.strip() # ---------------------------------------------------------------------- @staticmethod def _GenerateAdditionalDataChildren(element, exclude_names): children = OrderedDict() for child in element: if child.tag.startswith("_") or child.tag in exclude_names: continue children.setdefault(child.tag, []).append(child) if len(children) == 1 and next(six.iterkeys(children)) == "{collection_item_name}": value = next(six.itervalues(children)) if not isinstance(value, list): value = [value] yield None, value else: for k, v in six.iteritems(children): if len(v) == 1: yield k, v[0] else: yield k, v """, ).format( collection_item_name=Plugin.COLLECTION_ITEM_NAME, ) # ---------------------------------------------------------------------- @staticmethod @Interface.override def GetGlobalUtilityMethods(dest_writer): return textwrap.dedent( """\ # ---------------------------------------------------------------------- def _GetXmlElement( element, child_name, is_optional=False, is_collection=False, ): children = element.findall(child_name) if not children: if is_optional: return DoesNotExist raise SerializeException("No elements were found") if len(children) != 1: raise SerializeException("Multiple items were found ({{}})".format(len(children))) result = children[0] if is_collection: result = result.findall("{collection_item_name}") return result """, ).format( collection_item_name=Plugin.COLLECTION_ITEM_NAME, )
class Int32TypeInfoFactory(_ScalarTypeInfoFactory): TypeName = Interface.DerivedProperty("int32") CppType = Interface.DerivedProperty("std::int32_t") CType = Interface.DerivedProperty("int32_t")
class BoolTypeInfoFactory(_ScalarTypeInfoFactory): TypeName = Interface.DerivedProperty("bool") CppType = Interface.DerivedProperty("bool") CType = Interface.DerivedProperty("bool")
class VectorTypeInfoFactory(TypeInfoFactory): # ---------------------------------------------------------------------- # | # | Public Types # | # ---------------------------------------------------------------------- TypeName = Interface.DerivedProperty( re.compile(r"vector\<(?P<type>\S+)\>")) CppType = Interface.DerivedProperty(None) # ---------------------------------------------------------------------- # | # | Public Types # | # ---------------------------------------------------------------------- def __init__( self, custom_structs=None, custom_enums=None, member_type=None, create_type_info_factory_func=None, ): if member_type is not None: assert create_type_info_factory_func is not None match = self.TypeName.match(member_type) assert match, member_type the_type = match.group("type") type_info = create_type_info_factory_func(the_type) if not hasattr(type_info, "CType"): raise Exception( "'{}' is a type that can't be directly expressed in C and therefore cannot be used with a vector" .format(the_type)) self._type_info = type_info # Override the CppType property with this type info self.CppType = "std::tuple<{type} *, {type} *>".format( type=self._type_info.CppType, ) # ---------------------------------------------------------------------- @Interface.override def GetInputInfo(self, arg_name, is_optional, invocation_template): if is_optional: validation_statements = textwrap.dedent( """\ if({name}_ptr == nullptr && {name}_elements != 0) throw std::invalid_argument("'{name}_elements' is not 0"); if({name}_ptr != nullptr && {name}_elements == 0) throw std::invalid_argument("'{name}_elements' is 0"); """, ).format(name=arg_name, ) invocation_statements = invocation_template.format( "{name}_ptr ? std::make_tuple(const_cast<{type} *>({name}_ptr), const_cast<{type} *>({name}_ptr) + {name}_elements) : nonstd::optional<std::tuple<{type} const *, {type} const *>>()" .format( type=self._type_info.CType, name=arg_name, ), ) else: validation_statements = textwrap.dedent( """\ if({name}_ptr == nullptr) throw std::invalid_argument("'{name}_ptr' is null"); if({name}_elements == 0) throw std::invalid_argument("'{name}_elements' is 0"); """, ).format(name=arg_name, ) invocation_statements = invocation_template.format( "std::make_tuple(const_cast<{type} *>({name}_ptr), const_cast<{type} *>({name}_ptr) + {name}_elements)" .format( type=self._type_info.CType, name=arg_name, ), ) return self.Result( [ "/*in*/ {type} const *{name}_ptr".format( type=self._type_info.CType, name=arg_name, ), "/*in*/ size_t {}_elements".format(arg_name), ], validation_statements, invocation_statements, ) # ---------------------------------------------------------------------- @Interface.override def GetInputBufferInfo(self, arg_name, is_optional, invocation_template): if is_optional: raise Exception( "Optional vector values are not supported for input buffers") return self.Result( [ "/*in*/ {type} const **{name}_values_ptr".format( type=self._type_info.CType, name=arg_name, ), "/*in*/ size_t const *{}_sizes_ptr".format(arg_name), "/*in*/ size_t {}_elements".format(arg_name), ], textwrap.dedent( """\ if({name}_values_ptr == nullptr) throw std::invalid_argument("'{name}_values_ptr' is null"); if({name}_sizes_ptr == nullptr) throw std::invalid_argument("'{name}_sizes_ptr' is null"); if({name}_elements == 0) throw std::invalid_argument("'{name}_elements' is 0"); """, ).format(name=arg_name, ), textwrap.dedent( """\ std::vector<std::tuple<{type} *, {type} *>> {name}_buffer; {name}_buffer.reserve({name}_elements); {type} const * const * const {name}_values_end_ptr({name}_values_ptr + {name}_elements); while({name}_values_ptr != {name}_values_end_ptr) {{ {name}_buffer.emplace_back(const_cast<{type} *>(*{name}_values_ptr), const_cast<{type} *>(*{name}_values_ptr) + *{name}_sizes_ptr); ++{name}_values_ptr; ++{name}_sizes_ptr; }} {statement} """, ).format( type=self._type_info.CType, name=arg_name, statement=invocation_template.format( "{name}_buffer.data(), {name}_buffer.size()".format( name=arg_name, ), ), ), ) # ---------------------------------------------------------------------- @Interface.override def GetOutputInfo( self, arg_name, result_name="result", is_struct_member=False, ): return self.Result( [ "/*out*/ {type} **{name}_ptr".format( type=self._type_info.CType, name=arg_name, ), "/*out*/ size_t *{}_elements".format(arg_name), ], textwrap.dedent( """\ if({name}_ptr == nullptr) throw std::invalid_argument("'{name}_ptr' is null"); if({name}_elements == nullptr) throw std::invalid_argument("'{name}_elements' is null"); """, ).format(name=arg_name, ), # TODO: This code is assuming a result type of std::vector which # will likely change in the future. This code will have to be # updated when that change is made. textwrap.dedent( """\ if({result_name}.empty()) {{ {pointer}{name}_ptr = nullptr; {pointer}{name}_elements = 0; }} else {{ struct {name}Internal {{ static void Deleter({type} *pData) {{ delete [] pData; }} }}; std::unique_ptr<{type}, void (*)({type} *)> pBuffer(new {type}[{result_name}.size()], {name}Internal::Deleter); {type} * ptr(pBuffer.get()); for(auto &value : {result_name}) *ptr++ = std::move(value); {pointer}{name}_ptr = pBuffer.release(); {pointer}{name}_elements = {result_name}.size(); }} """, ).format( type=self._type_info.CType, name=arg_name, result_name=result_name, pointer="" if is_struct_member else "*", ), ) # ---------------------------------------------------------------------- @Interface.override def GetDestroyOutputInfo( self, arg_name="result", ): return self.Result( [ "/*in*/ {type} *{name}_ptr".format( type=self._type_info.CType, name=arg_name, ), "/*in*/ size_t {}_elements".format(arg_name), ], textwrap.dedent( """\ if({name}_ptr == nullptr && {name}_elements != 0) throw std::invalid_argument("Invalid buffer"); if({name}_ptr != nullptr && {name}_elements == 0) throw std::invalid_argument("Invalid buffer"); """, ).format(name=arg_name, ), textwrap.dedent( """\ if({name}_ptr) delete [] {name}_ptr; """, ).format(name=arg_name, ), )