class StringTypeInfo(TypeInfo):
# ----------------------------------------------------------------------
# |
# | Public Types
# |
# ----------------------------------------------------------------------
TypeName = Interface.DerivedProperty("string")
CppType = Interface.DerivedProperty("std::string")
CType = "char const *"
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
@Interface.override
def GetTransformInputArgs(
self,
input_name="input",
):
if self.IsOptional:
return "Microsoft::Featurizer::Traits<nonstd::optional<std::string>>::IsNull({input_name}) ? nullptr : {input_name}->c_str()".format(
input_name=input_name,
)
return "{}.c_str()".format(input_name)
# ----------------------------------------------------------------------
@Interface.override
def GetTransformInputBufferArgs(
self,
input_name='input',
):
raise NotImplementedError("Not implemented yet")
# ----------------------------------------------------------------------
@Interface.override
def GetOutputInfo(
self,
invocation_template,
result_name="result",
):
if self.IsOptional:
vector_type = "nonstd::optional<std::string>"
statement = "{result}_ptr ? std::string({result}_ptr) : nonstd::optional<std::string>()".format(
result=result_name,
)
else:
vector_type = "std::string"
statement = "{result}_ptr ? std::string({result}_ptr) : std::string()".format(
result=result_name,
)
return self.Result(
vector_type,
[self.Type("char const *", "{}_ptr".format(result_name))],
invocation_template.format(statement),
"{}_ptr".format(result_name),
destroy_inline=True,
)
class DatetimeTypeInfoFactory(TypeInfoFactory):
# ----------------------------------------------------------------------
# |
# | Public Types
# |
# ----------------------------------------------------------------------
TypeName = Interface.DerivedProperty("datetime")
CppType = Interface.DerivedProperty(
"std::chrono::system_clock::time_point")
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
@Interface.override
def GetTransformInputArgs(
self,
is_input_optional,
input_name="input",
):
if is_input_optional:
return "Microsoft::Featurizer::Traits<decltype({input_name})>::IsNull({input_name}) ? nullptr : CreateDateTimeParameter(*{input_name})".format(
input_name=input_name, )
return "CreateDateTimeParameter({input_name})".format(
input_name=input_name, )
# ----------------------------------------------------------------------
@Interface.override
def GetOutputInfo(
self,
result_name="result",
):
raise NotImplementedError("Not implemented yet")
class VectorTypeInfoFactory(TypeInfoFactory):
# ----------------------------------------------------------------------
# |
# | Public Properties
# |
# ----------------------------------------------------------------------
TypeName = Interface.DerivedProperty(
re.compile(r"vector\<(?P<type>\S+)\>"))
CSharpType = Interface.DerivedProperty("TODO1")
CSharpTypeName = Interface.DerivedProperty("TODO2")
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
def __init__(
self,
custom_structs=None,
custom_enums=None,
member_type=None,
create_type_info_factory_func=None,
):
if member_type is not None:
assert create_type_info_factory_func is not None
match = self.TypeName.match(member_type)
assert match, member_type
the_type = match.group("type")
type_info = create_type_info_factory_func(the_type)
self._type_info = type_info
# ----------------------------------------------------------------------
@Interface.override
def GetNativeInputInfo(self, is_optional):
return self.Result(
"TODO3: Parameter decl",
"TODO4: Validation statements",
"TODO5: Invocation statements",
"TODO6: Conversion end",
"TODO7: Delete transformed data",
)
# ----------------------------------------------------------------------
@Interface.override
def GetNativeOutputInfo(
self,
is_struct=False,
featurizer_name="",
):
return self.Result(
"TODO8: Parameter decl",
"TODO9: Validation statements",
"TODO10: Invocation statements",
"TODO11: Conversion end",
"TODO12: Delete transformed data",
)
class Plugin(PluginBase):
# ----------------------------------------------------------------------
# | Properties
Name = Interface.DerivedProperty("Debug")
Description = Interface.DerivedProperty(
"Plugin that displays debug information and nothing else", )
# ----------------------------------------------------------------------
# | Methods
@staticmethod
@Interface.override
def GetRequiredMetadataNames():
return []
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GenerateOutputFilenames(context):
assert "output_name" in context
yield "{}.txt".format(context["output_name"])
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def Execute(invoke_reason, context, status_stream, verbose_stream,
verbose): # <unused argument> pylint: disable = W0613
assert len(
context["output_filenames"]) == 1, context["output_filenames"]
output_filename = context["output_filenames"][0]
status_stream.write("Writing '{}'...".format(output_filename))
with status_stream.DoneManager():
sink = six.StringIO()
CommonEnvironment.Describe(
context,
output_stream=sink,
)
sink = sink.getvalue()
with open(output_filename, "w") as f:
f.write(
textwrap.dedent(
"""\
# ----------------------------------------------------------------------
# |
# | Debug Output - Execute
# |
# ----------------------------------------------------------------------
invoke_reason:
{invoke_reason}
context:
{context}
""", ).format(
invoke_reason=invoke_reason,
context=StringHelpers.LeftJustify(sink.strip(), 4),
), )
class Plugin(PluginBase):
# ----------------------------------------------------------------------
# | Properties
Name = Interface.DerivedProperty("SharedLibraryTests")
Description = Interface.DerivedProperty(
"Generates code used when testing the Shared Library import/export layer",
)
# ----------------------------------------------------------------------
# | Methods
@staticmethod
@Interface.override
def Generate(
open_file_func,
global_custom_structs,
global_custom_enums,
data,
output_dir,
status_stream,
):
result_code = 0
status_stream.write("Preprocessing data...")
with status_stream.DoneManager():
type_info_data = []
for items in data:
type_info_data.append([
TypeInfoData(item, global_custom_structs,
global_custom_enums) for item in items
])
for desc, func in [("Generating .h files...", _GenerateHeaderFile)]:
status_stream.write(desc)
with status_stream.DoneManager(suffix="\n", ) as dm:
for index, (items, items_type_info_data) in enumerate(
zip(data, type_info_data), ):
dm.stream.write(
"Processing '{}' ({} of {})...".format(
items[0].name,
index + 1,
len(data),
), )
with dm.stream.DoneManager() as this_dm:
this_dm.result = func(
open_file_func,
output_dir,
items,
items_type_info_data,
this_dm.stream,
)
if dm.result < 0:
return dm.result
result_code = result_code or dm.result
return result_code
class Plugin(PluginBase):
# ----------------------------------------------------------------------
# | Properties
Name = Interface.DerivedProperty("MLNet")
Description = Interface.DerivedProperty(
"Generates code used during the Shared Library import/export layer interfacing with the shared C++ functionality",
)
# ----------------------------------------------------------------------
# | Methods
@staticmethod
@Interface.override
def Generate(open_file_func, global_custom_structs, global_custom_enums,
data, output_dir, status_stream):
result_code = 0
status_stream.write("Preprocessing data for ML.NET...")
with status_stream.DoneManager():
# Convert the types into the corresponding C#/CPP types that will be used
# in the Shared Library interface.
csharp_data = []
unsupported_types = set()
for items in data:
csharp_data.append([
_FillList(item, status_stream, unsupported_types,
global_custom_structs, global_custom_enums)
for item in items
], )
for desc, func in [("Generating C# files...", _GenerateCSharpFile)]:
status_stream.write(desc)
with status_stream.DoneManager(suffix="\n", ) as dm:
for index, (items, items_csharp_data) in enumerate(
zip(data, csharp_data), ):
dm.stream.write(
"Processing '{}' ({} of {})...".format(
items[0].name,
index + 1,
len(data),
), )
with dm.stream.DoneManager() as this_dm:
this_dm.result = func(
open_file_func,
output_dir,
items,
items_csharp_data,
this_dm.stream,
)
if dm.result < 0:
return dm.result
result_code = result_code or dm.result
return result_code
class UniqueIdTypeInfo(TypeInfo):
# ----------------------------------------------------------------------
# |
# | Public Properties
# |
# ----------------------------------------------------------------------
TypeName = Interface.DerivedProperty("unique_id")
CSharpType = Interface.DerivedProperty("TODO1")
CSharpTypeName = Interface.DerivedProperty("TODO2")
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
def __init__(self,
*args,
member_type=None,
create_type_info_func=None,
**kwargs):
if member_type is None:
return
assert create_type_info_func is not None
super(UniqueIdTypeInfo, self).__init__(*args, **kwargs)
if self.IsOptional:
raise Exception("UniqueId types cannot be optional")
# ----------------------------------------------------------------------
@Interface.override
def GetNativeInputInfo(self):
return self.Result(
"TODO3: Parameter decl",
"TODO4: Validation statements",
"TODO5: Invocation statements",
"TODO6: Conversion end",
"TODO7: Delete transformed data",
)
# ----------------------------------------------------------------------
@Interface.override
def GetNativeOutputInfo(
self,
is_struct=False,
featurizer_name="",
):
return self.Result(
"TODO8: Parameter decl",
"TODO9: Validation statements",
"TODO10: Invocation statements",
"TODO11: Conversion end",
"TODO12: Delete transformed data",
)
class TestParser(TestParserImpl):
"""Parses content produced by Catch2"""
# ----------------------------------------------------------------------
# | Public Properties
Name = Interface.DerivedProperty("Catch2")
Description = Interface.DerivedProperty("Parses Catch2 output.")
# ----------------------------------------------------------------------
# | Public Methods
@staticmethod
@Interface.override
def IsSupportedCompiler(compiler):
# Many compilers are supported
return True
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def Parse(test_data):
if "All tests passed" not in test_data:
return -1
benchmark_data = ExtractBenchmarkOutput(test_data)
if benchmark_data:
return 0, benchmark_data
return 0
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def CreateInvokeCommandLine(context, debug_on_error):
if "output_filename" in context and context["output_filename"]:
output_filename = context["output_filename"]
elif "output_filenames" in context and context["output_filenames"]:
assert len(
context["output_filenames"]) == 1, context["output_filenames"]
output_filename = context["output_filenames"][0]
else:
assert False, context
dirname, basename = os.path.split(output_filename)
if context.get("is_benchmark", False):
flags = "[benchmark]"
else:
flags = "~[benchmark] --success"
return 'cd "{output_dir}" && {output_name} {flags}'.format(
output_dir=dirname,
output_name=basename,
flags=flags,
)
class StringTypeInfoFactory(TypeInfoFactory):
# ----------------------------------------------------------------------
# |
# | Public Types
# |
# ----------------------------------------------------------------------
TypeName = Interface.DerivedProperty("string")
CppType = Interface.DerivedProperty("std::string")
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
@Interface.override
def GetTransformInputArgs(
self,
is_input_optional,
input_name="input",
):
if is_input_optional:
return "Microsoft::Featurizer::Traits<nonstd::optional<std::string>>::IsNull({input_name}) ? nullptr : {input_name}->c_str()".format(
input_name=input_name, )
return "{}.c_str()".format(input_name)
# ----------------------------------------------------------------------
@Interface.override
def GetOutputInfo(
self,
result_name="result",
):
return self.Result(
"std::string",
textwrap.dedent(
"""\
char const * {result_name}_ptr(nullptr);
std::size_t {result_name}_items(0);
""", ).format(result_name=result_name, ),
"&{result_name}_ptr, &{result_name}_items".format(
result_name=result_name, ),
textwrap.dedent(
"""\
#if (defined __apple_build_version__)
results.push_back({result}_ptr ? std::string({result}_ptr) : std::string());
#else
results.emplace_back({result}_ptr ? std::string({result}_ptr) : std::string());
#endif
""", ).format(result=result_name, ),
"{result_name}_ptr, {result_name}_items".format(
result_name=result_name, ),
destroy_inline=True,
)
class Plugin(PluginBase):
"""Prints diagnostic information about the input code"""
# ----------------------------------------------------------------------
# | Properties
Name = Interface.DerivedProperty("Debug")
Priority = Interface.DerivedProperty(1)
# ----------------------------------------------------------------------
# | Methods
@classmethod
@Interface.override
def PreprocessLines(cls, lines):
return cls._Display("Pre clang-format", lines)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def Decorate(cls, lines):
return cls._Display("Post clang-format", lines)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def PostprocessLines(cls, lines):
return cls._Display("Post-Decoration", lines)
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
@staticmethod
def _Display(header, lines):
sys.stdout.write(
textwrap.dedent(
"""\
# ----------------------------------------------------------------------
# |
# | {}
# |
# ----------------------------------------------------------------------
""", ).format(header), )
for line_index, line in enumerate(lines):
sys.stdout.write("{0:>3}) {1}\n".format(line_index + 1, line))
sys.stdout.write("\n")
return lines
class DestinationStatementWriter(PythonDestinationStatementWriter):
# ----------------------------------------------------------------------
# | Public Properties
ObjectTypeDesc = Interface.DerivedProperty("a JSON object")
# ----------------------------------------------------------------------
# | Methods
@staticmethod
@Interface.override
def SerializeToString(var_name):
return "_JsonToString({var_name}, pretty_print)".format(
var_name=var_name, )
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetGlobalUtilityMethods(source_writer):
return textwrap.dedent(
"""\
# ----------------------------------------------------------------------
def _JsonToString(obj, pretty_print):
if pretty_print:
content = json.dumps(obj, cls=JsonEncoder, indent=2, separators=[", ", " : "])
# Remove trailing whitespace
return "\\n".join([line.rstrip() for line in content.split("\\n")])
else:
return json.dumps(obj, cls=JsonEncoder)
""", )
class SourceStatementWriter(PythonSourceStatementWriter):
# ----------------------------------------------------------------------
# | Properties
ObjectTypeDesc = Interface.DerivedProperty("a YAML object")
# ----------------------------------------------------------------------
# | Methods
@classmethod
@Interface.override
def ConvenienceConversions(cls, var_name, element_or_none):
content = textwrap.dedent(
"""\
if isinstance({var_name}, six.string_types):
if FileSystem.IsFilename({var_name}):
with open({var_name}) as f:
{var_name} = rtyaml.load(f)
else:
{var_name} = rtyaml.load({var_name})
""", ).format(var_name=var_name, )
if element_or_none is not None:
content += textwrap.dedent(
"""\
{}
""", ).format(
super(Plugin.SourceStatementWriter,
cls).ConvenienceConversions(
var_name, element_or_none))
return content
class TestExecutor(TestExecutorImpl):
# ----------------------------------------------------------------------
# | Properties
Name = Interface.DerivedProperty("ClangCodeCoverage")
Description = Interface.DerivedProperty(
"Extracts code coverage information using Clang tools.")
# ----------------------------------------------------------------------
# | Methods
@staticmethod
@Interface.override
def IsSupportedCompiler(compiler):
return compiler.Name in ["CMake"]
# ----------------------------------------------------------------------
_CodeCoverageExecutor = Interface.DerivedProperty(
ClangCodeCoverageExecutor)
class DestinationStatementWriter(PythonDestinationStatementWriter):
# ----------------------------------------------------------------------
# | Properties
ObjectTypeDesc = Interface.DerivedProperty("a YAML object")
# ----------------------------------------------------------------------
# | Methods
@staticmethod
@Interface.override
def SerializeToString(var_name):
return "rtyaml.dump({var_name})".format(var_name=var_name, )
class DatetimeTypeInfoFactory(TypeInfoFactory):
# ----------------------------------------------------------------------
# |
# | Public Properties
# |
# ----------------------------------------------------------------------
TypeName = Interface.DerivedProperty("datetime")
CSharpType = Interface.DerivedProperty("TODO")
CSharpTypeName = Interface.DerivedProperty("TODO")
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetNativeInputInfo(cls, is_optional):
return cls.Result(
"TODO: Parameter decl",
"TODO: Validation statements or None",
"TODO: invocation_statement",
"TODO: conversion end",
"TODO: delete transformed data statement or empty string",
)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetNativeOutputInfo(
cls,
is_struct=False,
featurizer_name="",
):
return cls.Result(
"TODO: Parameter decl",
"TODO: Validation statements or None",
"TODO: invocation_statement",
"TODO: conversion end",
"TODO: delete transformed data statement or empty string",
)
class TestExecutorImpl(TestExecutorImplBase):
# ----------------------------------------------------------------------
# | Methods
@staticmethod
@Interface.override
def ValidateEnvironment():
if CurrentShell.CategoryName != "Windows":
return "The '{}' test executor is only available on Windows".format(cls.Name)
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def IsSupportedCompiler(compiler):
return compiler.Name in ["CMake"]
# ----------------------------------------------------------------------
_CodeCoverageExecutor = Interface.DerivedProperty(MSVCCodeCoverageExecutor)
class DestinationStatementWriter(
PythonSerializationImpl.DestinationStatementWriter):
ObjectTypeDesc = Interface.DerivedProperty(
"an XML object (ElementTree)")
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def CreateCompoundElement(cls, element, attributes_var_or_none):
return textwrap.dedent(
"""\
_CreateXmlElement(
{name},
attributes={attributes},
)
""", ).format(
name=cls.GetElementStatementName(element),
attributes=attributes_var_or_none or "None",
)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def CreateSimpleElement(cls, element, attributes_var_or_none,
fundamental_statement):
return textwrap.dedent(
"""\
_CreateXmlElement(
{name},
attributes={attributes},
text_value={fundamental},
)
""", ).format(
name=cls.GetElementStatementName(element),
attributes=attributes_var_or_none or "None",
fundamental=fundamental_statement,
)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def CreateFundamentalElement(cls, element, fundamental_statement):
return textwrap.dedent(
"""\
_CreateXmlElement(
{name},
text_value={statement},
)
""", ).format(
name=cls.GetElementStatementName(element),
statement=fundamental_statement,
)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def CreateCollection(cls, element, result_name):
return "cls._CreateXmlCollection({}, {})".format(
cls.GetElementStatementName(element), result_name)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def AppendChild(cls, child_element, parent_var_name, var_name_or_none):
if getattr(child_element, "IsAttribute", False):
return "{parent_var_name}.attrib[{element_name}] = {var_name}".format(
parent_var_name=parent_var_name,
element_name=cls.GetElementStatementName(child_element),
var_name=var_name_or_none,
)
if var_name_or_none is None:
var_name_or_none = "_CreateXmlElement({})".format(
cls.GetElementStatementName(child_element))
return "{parent_var_name}.append({var_name})".format(
parent_var_name=parent_var_name,
var_name=var_name_or_none,
)
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def SerializeToString(var_name):
return textwrap.dedent(
"""\
ET.tostring(
_XmlPrettyPrint({var_name}) if pretty_print else {var_name},
encoding="utf-8",
method="xml",
).decode("utf-8")
""", ).format(var_name=var_name, )
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetClassUtilityMethods(source_writer):
return textwrap.dedent(
"""\
# ----------------------------------------------------------------------
@staticmethod
def _CreateXmlCollection(element_name, items_or_none):
result = _CreateXmlElement(element_name)
for item in (items_or_none or []):
item.tag = "{collection_item_name}"
result.append(item)
return result
""", ).format(
collection_item_name=Plugin.COLLECTION_ITEM_NAME, )
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetGlobalUtilityMethods(source_writer):
return textwrap.dedent(
"""\
# ----------------------------------------------------------------------
def _CreateXmlElement(
element_name,
attributes=None,
text_value=None,
):
result = ET.Element(
element_name,
attrib=attributes or {},
)
if text_value is not None:
result.text = text_value
return result
# ----------------------------------------------------------------------
def _XmlPrettyPrint(elem, level=0):
original = elem
i = "\\n" + level * " "
if elem:
if not elem.text or not elem.text.strip():
elem.text = i + " "
if not elem.tail or not elem.tail.strip():
elem.tail = i
for child in elem:
_XmlPrettyPrint(child, level + 1)
# <Using possibly undefined loop variable 'child'> pylint: disable = W0631
if not child.tail or not child.tail.strip():
child.tail = i
else:
if level and (not elem.tail or not elem.tail.strip()):
elem.tail = i
return original
""", )
class UInt64TypeInfoFactory(_ScalarTypeInfoFactory):
TypeName = Interface.DerivedProperty("uint64")
CppType = Interface.DerivedProperty("std::uint64_t")
CType = Interface.DerivedProperty("uint64_t")
class CodeGenerator(
AtomicInputProcessingMixin,
ConditionalInvocationQueryMixin,
MultipleOutputMixin,
CodeGeneratorBase,
):
# ----------------------------------------------------------------------
# | Types
ContextCode = namedtuple("ContextCode", ["filename", "var_name"])
# ----------------------------------------------------------------------
# | Properties
Name = Interface.DerivedProperty("Jinja2CodeGenerator")
Description = Interface.DerivedProperty(
"Processes a Jinja2 template and produces output")
InputTypeInfo = Interface.DerivedProperty(
CommandLine.FilenameTypeInfo(
validation_expression=".+?\.jinja2(?:\..+)?", ), )
# ----------------------------------------------------------------------
# | Methods
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def _GetOptionalMetadata(cls):
return [("jinja2_context", {}), ("jinja2_context_code", []),
("preserve_dir_structure", False), ("ignore_errors", False),
("debug", False)] + super(
CodeGenerator,
cls,
)._GetOptionalMetadata()
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def _GetRequiredMetadataNames(cls):
return ["output_dir"] + super(CodeGenerator,
cls)._GetRequiredMetadataNames()
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def _CreateContext(cls, metadata, status_stream):
jinja2_context = {}
# Load the custom context defined in code
for context_code in metadata["jinja2_context_code"]:
dirname, basename = os.path.split(context_code)
basename = os.path.splitext(basename)[0]
sys.path.insert(0, dirname)
with CallOnExit(lambda: sys.path.pop(0)):
mod = importlib.import_module(basename)
var = getattr(mod, context_code.var_name)
del mod
if isinstance(var, dict):
for k, v in six.iteritems(var):
jinja2_context[k] = v
else:
jinja2_context[context_code.var_name] = var
del metadata["jinja2_context_code"]
# Load the custom context
for k, v in six.iteritems(metadata["jinja2_context"]):
if len(v) == 1:
jinja2_context[k] = v[0]
else:
jinja2_context[k] = v
metadata["jinja2_context"] = jinja2_context
# Calculate the hashes of the input filenames. We will use this information
# during comparison to determine if an input file has changed. It appears
# that this value isn't used, but it is actually used when comparing the
# context of two different invocations.
# ----------------------------------------------------------------------
def CalculateHash(input_filename):
with open(input_filename, "rb") as f:
return hashlib.sha256(f.read()).digest()
# ----------------------------------------------------------------------
metadata["hashes"] = [
CalculateHash(input_filename)
for input_filename in metadata["inputs"]
]
# Get the output filenames
if not metadata["preserve_dir_structure"]:
# ----------------------------------------------------------------------
def GetBaseDir(input_filename):
return ''
# ----------------------------------------------------------------------
else:
if len(metadata["inputs"]) == 1:
common_prefix = os.path.dirname(metadata["inputs"][0])
else:
common_prefix = FileSystem.GetCommonPath(*metadata["inputs"])
# ----------------------------------------------------------------------
def GetBaseDir(input_filename):
return FileSystem.TrimPath(input_filename, common_prefix)
# ----------------------------------------------------------------------
output_filenames = []
for input_filename in metadata["inputs"]:
output_filenames.append(
os.path.join(
metadata["output_dir"],
GetBaseDir(input_filename),
'.'.join([
part
for part in os.path.basename(input_filename).split(".")
if part != "jinja2"
]),
), )
metadata["output_filenames"] = output_filenames
return super(CodeGenerator,
cls)._CreateContext(metadata, status_stream)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def _InvokeImpl(cls, invoke_reason, context, status_stream, verbose_stream,
verbose):
# ----------------------------------------------------------------------
class RelativeFileSystemLoader(FileSystemLoader):
# ----------------------------------------------------------------------
def __init__(
self,
input_filename,
searchpath=None,
*args,
**kwargs,
):
super(RelativeFileSystemLoader, self).__init__(
searchpath=[os.path.dirname(input_filename)] +
(searchpath or []),
*args,
**kwargs)
# ----------------------------------------------------------------------
def get_source(self, environment, template):
method = super(RelativeFileSystemLoader, self).get_source
try:
return method(environment, template)
except exceptions.TemplateNotFound:
for searchpath in reversed(self.searchpath):
potential_template = os.path.normpath(
os.path.join(searchpath,
template).replace('/', os.path.sep))
if os.path.isfile(potential_template):
dirname, basename = os.path.split(
potential_template)
self.searchpath.append(dirname)
return method(environment, template)
raise
# ----------------------------------------------------------------------
with status_stream.DoneManager(display=False, ) as dm:
for index, (input_filename, output_filename) in enumerate(
zip(
context["inputs"],
context["output_filenames"],
)):
status_stream.write(
"Processing '{}' ({} of {})...".format(
input_filename,
index + 1,
len(context["inputs"]),
), )
with dm.stream.DoneManager(
suppress_exceptions=True, ) as this_dm:
try:
# ----------------------------------------------------------------------
def ReadFileFilter(value):
potential_filename = os.path.join(
os.path.dirname(input_filename), value)
if not os.path.isfile(potential_filename):
return "<< '{}' was not found >>".format(
potential_filename)
with open(potential_filename) as f:
return f.read()
# ----------------------------------------------------------------------
loader = RelativeFileSystemLoader(input_filename)
if context["debug"]:
from jinja2 import meta
env = Environment(loader=loader, )
with open(input_filename) as f:
content = env.parse(f.read())
this_dm.stream.write("Variables:\n{}\n".format(
"\n".join([
" - {}".format(var) for var in
meta.find_undeclared_variables(content)
])))
continue
if context["ignore_errors"]:
undef = Undefined
else:
undef = StrictUndefined
env = Environment(
trim_blocks=True,
lstrip_blocks=True,
loader=loader,
undefined=undef,
)
env.tests["valid_file"] = lambda value: os.path.isfile(
os.path.dirname(input_filename), value)
env.filters[
"doubleslash"] = lambda value: value.replace(
"\\", "\\\\")
# Technically speaking, this isn't required as Jinja's import/include/extend functionality
# superseeds this functionality. However, it remains in the name of backwards compatibility.
env.filters["read_file"] = ReadFileFilter
with open(input_filename) as f:
template = env.from_string(f.read())
try:
content = template.render(
**context["jinja2_context"])
except exceptions.UndefinedError as ex:
this_dm.stream.write("ERROR: {}\n".format(str(ex)))
this_dm.result = -1
continue
with open(output_filename, "w") as f:
f.write(content)
except:
this_dm.result = -1
raise
return dm.result
class Plugin(PythonSerializationImpl):
# ----------------------------------------------------------------------
# | Properties
Name = Interface.DerivedProperty("PythonYaml")
Description = Interface.DerivedProperty(
"Creates python code that is able to serialize and deserialize python objects to YAML"
)
# ----------------------------------------------------------------------
# | Methods
@classmethod
@Interface.override
def GetAdditionalGeneratorItems(cls, context):
return [
_script_fullpath, PythonDestinationStatementWriter,
PythonSourceStatementWriter
] + super(Plugin, cls).GetAdditionalGeneratorItems(context)
# ----------------------------------------------------------------------
# | Private Types
@Interface.staticderived
class SourceStatementWriter(PythonSourceStatementWriter):
# ----------------------------------------------------------------------
# | Properties
ObjectTypeDesc = Interface.DerivedProperty("a YAML object")
# ----------------------------------------------------------------------
# | Methods
@classmethod
@Interface.override
def ConvenienceConversions(cls, var_name, element_or_none):
content = textwrap.dedent(
"""\
if isinstance({var_name}, six.string_types):
if FileSystem.IsFilename({var_name}):
with open({var_name}) as f:
{var_name} = rtyaml.load(f)
else:
{var_name} = rtyaml.load({var_name})
""", ).format(var_name=var_name, )
if element_or_none is not None:
content += textwrap.dedent(
"""\
{}
""", ).format(
super(Plugin.SourceStatementWriter,
cls).ConvenienceConversions(
var_name, element_or_none))
return content
# ----------------------------------------------------------------------
@Interface.staticderived
class DestinationStatementWriter(PythonDestinationStatementWriter):
# ----------------------------------------------------------------------
# | Properties
ObjectTypeDesc = Interface.DerivedProperty("a YAML object")
# ----------------------------------------------------------------------
# | Methods
@staticmethod
@Interface.override
def SerializeToString(var_name):
return "rtyaml.dump({var_name})".format(var_name=var_name, )
# ----------------------------------------------------------------------
# | Private Properties
_SupportAttributes = Interface.DerivedProperty(False)
_SupportAnyElements = Interface.DerivedProperty(True)
_SupportDictionaryElements = Interface.DerivedProperty(True)
_TypeInfoSerializationName = Interface.DerivedProperty("YamlSerialization")
_SourceStatementWriter = Interface.DerivedProperty(SourceStatementWriter)
_DestinationStatementWriter = Interface.DerivedProperty(
DestinationStatementWriter)
# ----------------------------------------------------------------------
# | Private Methods
@staticmethod
@Interface.override
def _WriteFileHeader(output_stream):
output_stream.write(
textwrap.dedent(
"""\
import rtyaml
from CommonEnvironment.CallOnExit import CallOnExit
from CommonEnvironment import FileSystem
from CommonEnvironment.TypeInfo.FundamentalTypes.Serialization.YamlSerialization import YamlSerialization
""", ), )
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def _WriteFileFooter(output_stream):
output_stream.write(
textwrap.dedent(
"""\
# ----------------------------------------------------------------------
def _ObjectToYaml(dumper, data):
d = dict(data.__dict__)
for k in list(six.iterkeys(d)):
if k.startswith("_"):
del d[k]
return dumper.represent_dict(d)
rtyaml.Dumper.add_representer(Object, _ObjectToYaml)
""", ), )
class FloatTypeInfoFactory(_ScalarTypeInfoFactory):
TypeName = Interface.DerivedProperty("float")
CppType = Interface.DerivedProperty("std::float_t")
CType = Interface.DerivedProperty("float")
class PythonSourceStatementWriter(SourceStatementWriter):
ObjectTypeDesc = Interface.DerivedProperty("a python object")
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def ConvenienceConversions(cls, var_name, element_or_none):
if element_or_none is None:
return ""
return textwrap.dedent(
"""\
if not isinstance({var_name}, list):
if isinstance({var_name}, dict) and "{name}" in {var_name}:
{var_name} = {var_name}["{name}"]
elif not isinstance({var_name}, dict) and hasattr({var_name}, "{name}"):
{var_name} = getattr({var_name}, "{name}")
elif is_root:
{var_name} = DoesNotExist
""",
).format(
var_name=var_name,
name=element_or_none.Name,
)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetChild(
cls,
var_name,
child_element,
is_simple_schema_fundamental=False,
):
if is_simple_schema_fundamental:
is_optional = False
else:
is_optional = child_element.TypeInfo.Arity.Min == 0
return textwrap.dedent(
"""\
cls._GetPythonAttribute(
{var_name},
{name},
is_optional={is_optional},
)
""",
).format(
var_name=var_name,
name=cls.GetElementStatementName(child_element),
is_optional=is_optional,
)
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetFundamental(var_name, child_element):
return var_name
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetAdditionalDataChildren():
return '[(k, v) for k, v in six.iteritems(source if isinstance(source, dict) else getattr(source, "__dict__", {})) if not k.startswith("_") and k not in exclude_names]'
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def CreateAdditionalDataItem(cls, dest_writer, name_var_name, source_var_name):
temporary_element = cls.CreateTemporaryElement(name_var_name, "1")
temporary_children_element = cls.CreateTemporaryElement("k", "+")
return textwrap.dedent(
"""\
# The following types should be returned directly without additional conversion
if isinstance({source_var_name}, (int, float, str, bool)):
return {source_var_name}
assert not isinstance({source_var_name}, list), {source_var_name}
if not isinstance({source_var_name}, dict):
{source_var_name} = {source_var_name}.__dict__
source_attribute_names = {source_var_name}.get("{attribute_names}", set())
attributes = OrderedDict()
items = OrderedDict()
for k, v in six.iteritems(source):
if k.startswith("_"):
continue
if k in source_attribute_names:
attributes[k] = v
else:
items[k] = v
if len(items) == 1 and next(six.iterkeys(items)) == {source_var_name}.get("{fundamental_name}", None):
return {simple_statement}
result = {compound_statement}
for k, v in six.iteritems(items):
try:
if isinstance(v, list):
new_items = []
for index, child in enumerate(v):
try:
new_items.append(cls._CreateAdditionalDataItem("item", child))
except:
_DecorateActiveException("Index {{}}".format(index))
{append_children}
else:
new_item = cls._CreateAdditionalDataItem(k, v)
{append_child}
except:
_DecorateActiveException(k)
return result
""",
).format(
source_var_name=source_var_name,
attribute_names=cls.ATTRIBUTES_ATTRIBUTE_NAME,
fundamental_name=cls.SIMPLE_ELEMENT_FUNDAMENTAL_ATTRIBUTE_NAME,
simple_statement=StringHelpers.LeftJustify(
dest_writer.CreateSimpleElement(
temporary_element,
"attributes",
'{}[{}["{}"]]'.format(source_var_name, source_var_name, cls.SIMPLE_ELEMENT_FUNDAMENTAL_ATTRIBUTE_NAME),
),
4,
).strip(),
compound_statement=dest_writer.CreateCompoundElement(temporary_element, "attributes").strip(),
append_children=StringHelpers.LeftJustify(
dest_writer.AppendChild(temporary_children_element, "result", dest_writer.CreateCollection(temporary_children_element, "new_items")),
12,
).strip(),
append_child=StringHelpers.LeftJustify(dest_writer.AppendChild(cls.CreateTemporaryElement("k", "1"), "result", "new_item"), 8).strip(),
)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetGlobalUtilityMethods(cls, dest_writer):
return PythonStatementWriterMixin.GetGlobalUtilityMethods(cls.ATTRIBUTES_ATTRIBUTE_NAME)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetClassUtilityMethods(cls, dest_writer):
return textwrap.dedent(
"""\
# ----------------------------------------------------------------------
@staticmethod
def _GetPythonAttribute(
item,
attribute_name,
is_optional=False,
):
if not isinstance(item, dict):
if hasattr(item, "__dict__"):
item = item.__dict__
else:
item = {}
value = item.get(attribute_name, DoesNotExist)
if value is DoesNotExist and not is_optional:
raise SerializeException("No items were found")
return value
""",
)
class DoubleTypeInfoFactory(_ScalarTypeInfoFactory):
TypeName = Interface.DerivedProperty("double")
CppType = Interface.DerivedProperty("std::double_t")
CType = Interface.DerivedProperty("double")
class Plugin(PythonSerializationImpl):
# ----------------------------------------------------------------------
# | Properties
Name = Interface.DerivedProperty("PythonJson")
Description = Interface.DerivedProperty(
"Creates python code that is able to serialize and deserialize python objects to JSON"
)
# ----------------------------------------------------------------------
# | Methods
@classmethod
@Interface.override
def GetAdditionalGeneratorItems(cls, context):
return [
_script_fullpath, PythonDestinationStatementWriter,
PythonSourceStatementWriter
] + super(Plugin, cls).GetAdditionalGeneratorItems(context)
# ----------------------------------------------------------------------
# | Private Types
@Interface.staticderived
class SourceStatementWriter(PythonSourceStatementWriter):
# ----------------------------------------------------------------------
# | Public Properties
ObjectTypeDesc = Interface.DerivedProperty("a JSON object")
# ----------------------------------------------------------------------
# | Methods
@classmethod
@Interface.override
def ConvenienceConversions(cls, var_name, element_or_none):
content = textwrap.dedent(
"""\
if isinstance({var_name}, six.string_types):
if FileSystem.IsFilename({var_name}):
with open({var_name}) as f:
{var_name} = json.load(f)
else:
{var_name} = json.loads({var_name})
""", ).format(var_name=var_name, )
if element_or_none is not None:
content += textwrap.dedent(
"""\
{}
""", ).format(
super(Plugin.SourceStatementWriter,
cls).ConvenienceConversions(
var_name, element_or_none))
return content
# ----------------------------------------------------------------------
@Interface.staticderived
class DestinationStatementWriter(PythonDestinationStatementWriter):
# ----------------------------------------------------------------------
# | Public Properties
ObjectTypeDesc = Interface.DerivedProperty("a JSON object")
# ----------------------------------------------------------------------
# | Methods
@staticmethod
@Interface.override
def SerializeToString(var_name):
return "_JsonToString({var_name}, pretty_print)".format(
var_name=var_name, )
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetGlobalUtilityMethods(source_writer):
return textwrap.dedent(
"""\
# ----------------------------------------------------------------------
def _JsonToString(obj, pretty_print):
if pretty_print:
content = json.dumps(obj, cls=JsonEncoder, indent=2, separators=[", ", " : "])
# Remove trailing whitespace
return "\\n".join([line.rstrip() for line in content.split("\\n")])
else:
return json.dumps(obj, cls=JsonEncoder)
""", )
# ----------------------------------------------------------------------
# | Private Properties
_SupportAttributes = Interface.DerivedProperty(False)
_SupportAnyElements = Interface.DerivedProperty(True)
_SupportDictionaryElements = Interface.DerivedProperty(True)
_TypeInfoSerializationName = Interface.DerivedProperty("JsonSerialization")
_SourceStatementWriter = Interface.DerivedProperty(SourceStatementWriter)
_DestinationStatementWriter = Interface.DerivedProperty(
DestinationStatementWriter)
# ----------------------------------------------------------------------
# | Private Methods
@staticmethod
@Interface.override
def _WriteFileHeader(output_stream):
output_stream.write(
textwrap.dedent(
"""\
import json
from CommonEnvironment import FileSystem
from CommonEnvironment.TypeInfo.FundamentalTypes.Serialization.JsonSerialization import JsonSerialization
# ----------------------------------------------------------------------
class JsonEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, Object):
d = copy.deepcopy(o.__dict__)
for k in list(six.iterkeys(d)):
if k.startswith("_"):
del d[k]
return d
return getattr(o, "__dict__", o)
""", ), )
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def _WriteFileFooter(output_stream):
# Nothing to do here
pass
class Plugin(PythonSerializationImpl):
COLLECTION_ITEM_NAME = "item"
# ----------------------------------------------------------------------
# | Properties
Name = Interface.DerivedProperty("PythonXml")
Description = Interface.DerivedProperty(
"Creates Python code that is able to serialize and deserialize python objects to XML"
)
# ----------------------------------------------------------------------
# | Methods
@classmethod
@Interface.override
def GetAdditionalGeneratorItems(cls, context):
return [_script_fullpath] + super(
Plugin, cls).GetAdditionalGeneratorItems(context)
# ----------------------------------------------------------------------
# | Private Types
@Interface.staticderived
class SourceStatementWriter(PythonSerializationImpl.SourceStatementWriter):
ObjectTypeDesc = Interface.DerivedProperty(
"an XML object (ElementTree)")
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def ConvenienceConversions(var_name, element_or_none):
content = textwrap.dedent(
"""\
if isinstance({var_name}, six.string_types):
if FileSystem.IsFilename({var_name}):
with open({var_name}) as f:
{var_name} = f.read()
{var_name} = ET.fromstring({var_name})
""", ).format(var_name=var_name, )
if element_or_none is not None:
content += textwrap.dedent(
"""\
potential_child = _GetXmlElement(
{var_name},
"{name}",
is_optional=True,
is_collection={is_collection},
)
if potential_child is not DoesNotExist or is_root:
{var_name} = potential_child
""",
).format(
var_name=var_name,
name=element_or_none.Name,
is_optional=element_or_none.TypeInfo.Arity.Min == 0,
is_collection=element_or_none.TypeInfo.Arity.IsCollection,
)
return content
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetChild(
cls,
var_name,
child_element,
is_simple_schema_fundamental=False,
):
if is_simple_schema_fundamental:
return cls.GetFundamental(var_name, child_element)
if getattr(child_element, "IsAttribute", False):
return textwrap.dedent(
"""\
cls._GetXmlAttribute(
{var_name},
{name},
is_optional={is_optional},
)
""", ).format(
var_name=var_name,
name=cls.GetElementStatementName(child_element),
is_optional=child_element.TypeInfo.Arity.Min == 0,
)
return textwrap.dedent(
"""\
_GetXmlElement(
{var_name},
{name},
is_optional={is_optional},
is_collection={is_collection},
)
""", ).format(
var_name=var_name,
name=cls.GetElementStatementName(child_element),
is_optional=child_element.TypeInfo.Arity.Min == 0,
is_collection=child_element.TypeInfo.Arity.IsCollection,
)
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetFundamental(var_name, child_element):
if getattr(child_element, "IsAttribute", False):
return var_name
return "cls.GetText({})".format(var_name)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetAdditionalDataChildren(cls):
return "cls._GenerateAdditionalDataChildren(source, exclude_names)"
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def CreateAdditionalDataItem(cls, dest_writer, name_var_name,
source_var_name):
temporary_element = cls.CreateTemporaryElement(
"{}.tag".format(source_var_name), "1")
return textwrap.dedent(
"""\
attributes = OrderedDict()
for k, v in six.iteritems({source_var_name}.attrib):
if k.startswith("_"):
continue
attributes[k] = v
if {source_var_name}.text and {source_var_name}.text.strip() and not {source_var_name}:
return {simple_element}
result = {compound_statement}
for child_name, child_or_children in cls._GenerateAdditionalDataChildren({source_var_name}, set()):
try:
if isinstance(child_or_children, list):
new_items = []
for index, child in enumerate(child_or_children):
try:
new_items.append(cls._CreateAdditionalDataItem("{collection_item_name}", child))
except:
_DecorateActiveException("Index {{}}".format(index))
if child_name is None:
result = new_items
break
{append_children}
else:
assert child_name is not None
new_item = cls._CreateAdditionalDataItem(child_name, child_or_children)
{append_child}
except:
_DecorateActiveException(child_name)
return result
""", ).format(
source_var_name=source_var_name,
collection_item_name=Plugin.COLLECTION_ITEM_NAME,
compound_statement=dest_writer.CreateCompoundElement(
temporary_element, "attributes").strip(),
simple_element=StringHelpers.LeftJustify(
dest_writer.CreateSimpleElement(
temporary_element, "attributes",
"{}.text".format(source_var_name)), 4).strip(),
append_children=StringHelpers.LeftJustify(
dest_writer.AppendChild(
cls.CreateTemporaryElement("child_name", "+"),
"result", "new_items"), 12).strip(),
append_child=StringHelpers.LeftJustify(
dest_writer.AppendChild(
cls.CreateTemporaryElement("child_name", "1"),
"result", "new_item"), 8).strip(),
)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetClassUtilityMethods(cls, dest_writer):
return textwrap.dedent(
"""\
# ----------------------------------------------------------------------
@staticmethod
def _GetXmlAttribute(
element,
attribute_name,
is_optional=False,
):
value = element.attrib.get(attribute_name, DoesNotExist)
if value is DoesNotExist and not is_optional:
raise SerializeException("The attribute '{{}}' does not exist".format(attribute_name))
return value
# ----------------------------------------------------------------------
@staticmethod
def GetText(item):
if item is None:
item = ""
if hasattr(item, "text"):
item = item.text
return item.strip()
# ----------------------------------------------------------------------
@staticmethod
def _GenerateAdditionalDataChildren(element, exclude_names):
children = OrderedDict()
for child in element:
if child.tag.startswith("_") or child.tag in exclude_names:
continue
children.setdefault(child.tag, []).append(child)
if len(children) == 1 and next(six.iterkeys(children)) == "{collection_item_name}":
value = next(six.itervalues(children))
if not isinstance(value, list):
value = [value]
yield None, value
else:
for k, v in six.iteritems(children):
if len(v) == 1:
yield k, v[0]
else:
yield k, v
""", ).format(
collection_item_name=Plugin.COLLECTION_ITEM_NAME, )
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetGlobalUtilityMethods(dest_writer):
return textwrap.dedent(
"""\
# ----------------------------------------------------------------------
def _GetXmlElement(
element,
child_name,
is_optional=False,
is_collection=False,
):
children = element.findall(child_name)
if not children:
if is_optional:
return DoesNotExist
raise SerializeException("No elements were found")
if len(children) != 1:
raise SerializeException("Multiple items were found ({{}})".format(len(children)))
result = children[0]
if is_collection:
result = result.findall("{collection_item_name}")
return result
""", ).format(
collection_item_name=Plugin.COLLECTION_ITEM_NAME, )
# ----------------------------------------------------------------------
@Interface.staticderived
class DestinationStatementWriter(
PythonSerializationImpl.DestinationStatementWriter):
ObjectTypeDesc = Interface.DerivedProperty(
"an XML object (ElementTree)")
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def CreateCompoundElement(cls, element, attributes_var_or_none):
return textwrap.dedent(
"""\
_CreateXmlElement(
{name},
attributes={attributes},
)
""", ).format(
name=cls.GetElementStatementName(element),
attributes=attributes_var_or_none or "None",
)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def CreateSimpleElement(cls, element, attributes_var_or_none,
fundamental_statement):
return textwrap.dedent(
"""\
_CreateXmlElement(
{name},
attributes={attributes},
text_value={fundamental},
)
""", ).format(
name=cls.GetElementStatementName(element),
attributes=attributes_var_or_none or "None",
fundamental=fundamental_statement,
)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def CreateFundamentalElement(cls, element, fundamental_statement):
return textwrap.dedent(
"""\
_CreateXmlElement(
{name},
text_value={statement},
)
""", ).format(
name=cls.GetElementStatementName(element),
statement=fundamental_statement,
)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def CreateCollection(cls, element, result_name):
return "cls._CreateXmlCollection({}, {})".format(
cls.GetElementStatementName(element), result_name)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def AppendChild(cls, child_element, parent_var_name, var_name_or_none):
if getattr(child_element, "IsAttribute", False):
return "{parent_var_name}.attrib[{element_name}] = {var_name}".format(
parent_var_name=parent_var_name,
element_name=cls.GetElementStatementName(child_element),
var_name=var_name_or_none,
)
if var_name_or_none is None:
var_name_or_none = "_CreateXmlElement({})".format(
cls.GetElementStatementName(child_element))
return "{parent_var_name}.append({var_name})".format(
parent_var_name=parent_var_name,
var_name=var_name_or_none,
)
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def SerializeToString(var_name):
return textwrap.dedent(
"""\
ET.tostring(
_XmlPrettyPrint({var_name}) if pretty_print else {var_name},
encoding="utf-8",
method="xml",
).decode("utf-8")
""", ).format(var_name=var_name, )
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetClassUtilityMethods(source_writer):
return textwrap.dedent(
"""\
# ----------------------------------------------------------------------
@staticmethod
def _CreateXmlCollection(element_name, items_or_none):
result = _CreateXmlElement(element_name)
for item in (items_or_none or []):
item.tag = "{collection_item_name}"
result.append(item)
return result
""", ).format(
collection_item_name=Plugin.COLLECTION_ITEM_NAME, )
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetGlobalUtilityMethods(source_writer):
return textwrap.dedent(
"""\
# ----------------------------------------------------------------------
def _CreateXmlElement(
element_name,
attributes=None,
text_value=None,
):
result = ET.Element(
element_name,
attrib=attributes or {},
)
if text_value is not None:
result.text = text_value
return result
# ----------------------------------------------------------------------
def _XmlPrettyPrint(elem, level=0):
original = elem
i = "\\n" + level * " "
if elem:
if not elem.text or not elem.text.strip():
elem.text = i + " "
if not elem.tail or not elem.tail.strip():
elem.tail = i
for child in elem:
_XmlPrettyPrint(child, level + 1)
# <Using possibly undefined loop variable 'child'> pylint: disable = W0631
if not child.tail or not child.tail.strip():
child.tail = i
else:
if level and (not elem.tail or not elem.tail.strip()):
elem.tail = i
return original
""", )
# ----------------------------------------------------------------------
# | Private Properties
_SupportAttributes = Interface.DerivedProperty(True)
_SupportAnyElements = Interface.DerivedProperty(True)
_SupportDictionaryElements = Interface.DerivedProperty(False)
_TypeInfoSerializationName = Interface.DerivedProperty("XmlSerialization")
_SourceStatementWriter = Interface.DerivedProperty(SourceStatementWriter)
_DestinationStatementWriter = Interface.DerivedProperty(
DestinationStatementWriter)
# ----------------------------------------------------------------------
# | Private Methods
@staticmethod
@Interface.override
def _WriteFileHeader(output_stream):
output_stream.write(
textwrap.dedent(
"""\
import xml.etree.ElementTree as ET
from CommonEnvironment import FileSystem
from CommonEnvironment.TypeInfo.FundamentalTypes.Serialization.XmlSerialization import XmlSerialization
""", ), )
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def _WriteFileFooter(output_stream):
# Nothing to do here
pass
class Plugin(RelationalPluginImpl):
# ----------------------------------------------------------------------
# |
# | Public Properties
# |
# ----------------------------------------------------------------------
Name = Interface.DerivedProperty("DebugRelational")
Description = Interface.DerivedProperty(
"Displays information associated with objects created by the base Relational plugin class"
)
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetAdditionalGeneratorItems(cls, context):
return super(Plugin, cls).GetAdditionalGeneratorItems(context) + [
RelationalPluginImpl
]
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GenerateOutputFilenames(
cls,
context,
all_objects=None,
):
return [
os.path.join(context["output_dir"],
"{}.txt".format(context["output_name"]))
]
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def Generate(
cls,
simple_schema_generator,
invoke_reason,
input_filenames,
output_filenames,
name,
elements,
include_indexes,
status_stream,
verbose_stream,
verbose,
):
with open(output_filenames[0], "w") as f:
f.write(cls._GenerateFileHeader())
for obj in cls.AllObjects:
f.write(
textwrap.dedent(
"""\
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
Unique Name: {unique}
Singular Name: {singular}
Plural Name: {plural}
""", ).format(
unique=obj.UniqueName,
singular=obj.SingularName,
plural=obj.PluralName,
), )
for child in obj.children:
f.write(
textwrap.dedent(
"""\
# ----------------------------------------------------------------------
{}
{}
""", ).format(
child.Name,
child.Item,
), )
f.write("\n")
class SourceStatementWriter(PythonSerializationImpl.SourceStatementWriter):
ObjectTypeDesc = Interface.DerivedProperty(
"an XML object (ElementTree)")
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def ConvenienceConversions(var_name, element_or_none):
content = textwrap.dedent(
"""\
if isinstance({var_name}, six.string_types):
if FileSystem.IsFilename({var_name}):
with open({var_name}) as f:
{var_name} = f.read()
{var_name} = ET.fromstring({var_name})
""", ).format(var_name=var_name, )
if element_or_none is not None:
content += textwrap.dedent(
"""\
potential_child = _GetXmlElement(
{var_name},
"{name}",
is_optional=True,
is_collection={is_collection},
)
if potential_child is not DoesNotExist or is_root:
{var_name} = potential_child
""",
).format(
var_name=var_name,
name=element_or_none.Name,
is_optional=element_or_none.TypeInfo.Arity.Min == 0,
is_collection=element_or_none.TypeInfo.Arity.IsCollection,
)
return content
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetChild(
cls,
var_name,
child_element,
is_simple_schema_fundamental=False,
):
if is_simple_schema_fundamental:
return cls.GetFundamental(var_name, child_element)
if getattr(child_element, "IsAttribute", False):
return textwrap.dedent(
"""\
cls._GetXmlAttribute(
{var_name},
{name},
is_optional={is_optional},
)
""", ).format(
var_name=var_name,
name=cls.GetElementStatementName(child_element),
is_optional=child_element.TypeInfo.Arity.Min == 0,
)
return textwrap.dedent(
"""\
_GetXmlElement(
{var_name},
{name},
is_optional={is_optional},
is_collection={is_collection},
)
""", ).format(
var_name=var_name,
name=cls.GetElementStatementName(child_element),
is_optional=child_element.TypeInfo.Arity.Min == 0,
is_collection=child_element.TypeInfo.Arity.IsCollection,
)
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetFundamental(var_name, child_element):
if getattr(child_element, "IsAttribute", False):
return var_name
return "cls.GetText({})".format(var_name)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetAdditionalDataChildren(cls):
return "cls._GenerateAdditionalDataChildren(source, exclude_names)"
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def CreateAdditionalDataItem(cls, dest_writer, name_var_name,
source_var_name):
temporary_element = cls.CreateTemporaryElement(
"{}.tag".format(source_var_name), "1")
return textwrap.dedent(
"""\
attributes = OrderedDict()
for k, v in six.iteritems({source_var_name}.attrib):
if k.startswith("_"):
continue
attributes[k] = v
if {source_var_name}.text and {source_var_name}.text.strip() and not {source_var_name}:
return {simple_element}
result = {compound_statement}
for child_name, child_or_children in cls._GenerateAdditionalDataChildren({source_var_name}, set()):
try:
if isinstance(child_or_children, list):
new_items = []
for index, child in enumerate(child_or_children):
try:
new_items.append(cls._CreateAdditionalDataItem("{collection_item_name}", child))
except:
_DecorateActiveException("Index {{}}".format(index))
if child_name is None:
result = new_items
break
{append_children}
else:
assert child_name is not None
new_item = cls._CreateAdditionalDataItem(child_name, child_or_children)
{append_child}
except:
_DecorateActiveException(child_name)
return result
""", ).format(
source_var_name=source_var_name,
collection_item_name=Plugin.COLLECTION_ITEM_NAME,
compound_statement=dest_writer.CreateCompoundElement(
temporary_element, "attributes").strip(),
simple_element=StringHelpers.LeftJustify(
dest_writer.CreateSimpleElement(
temporary_element, "attributes",
"{}.text".format(source_var_name)), 4).strip(),
append_children=StringHelpers.LeftJustify(
dest_writer.AppendChild(
cls.CreateTemporaryElement("child_name", "+"),
"result", "new_items"), 12).strip(),
append_child=StringHelpers.LeftJustify(
dest_writer.AppendChild(
cls.CreateTemporaryElement("child_name", "1"),
"result", "new_item"), 8).strip(),
)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetClassUtilityMethods(cls, dest_writer):
return textwrap.dedent(
"""\
# ----------------------------------------------------------------------
@staticmethod
def _GetXmlAttribute(
element,
attribute_name,
is_optional=False,
):
value = element.attrib.get(attribute_name, DoesNotExist)
if value is DoesNotExist and not is_optional:
raise SerializeException("The attribute '{{}}' does not exist".format(attribute_name))
return value
# ----------------------------------------------------------------------
@staticmethod
def GetText(item):
if item is None:
item = ""
if hasattr(item, "text"):
item = item.text
return item.strip()
# ----------------------------------------------------------------------
@staticmethod
def _GenerateAdditionalDataChildren(element, exclude_names):
children = OrderedDict()
for child in element:
if child.tag.startswith("_") or child.tag in exclude_names:
continue
children.setdefault(child.tag, []).append(child)
if len(children) == 1 and next(six.iterkeys(children)) == "{collection_item_name}":
value = next(six.itervalues(children))
if not isinstance(value, list):
value = [value]
yield None, value
else:
for k, v in six.iteritems(children):
if len(v) == 1:
yield k, v[0]
else:
yield k, v
""", ).format(
collection_item_name=Plugin.COLLECTION_ITEM_NAME, )
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetGlobalUtilityMethods(dest_writer):
return textwrap.dedent(
"""\
# ----------------------------------------------------------------------
def _GetXmlElement(
element,
child_name,
is_optional=False,
is_collection=False,
):
children = element.findall(child_name)
if not children:
if is_optional:
return DoesNotExist
raise SerializeException("No elements were found")
if len(children) != 1:
raise SerializeException("Multiple items were found ({{}})".format(len(children)))
result = children[0]
if is_collection:
result = result.findall("{collection_item_name}")
return result
""", ).format(
collection_item_name=Plugin.COLLECTION_ITEM_NAME, )
class Int32TypeInfoFactory(_ScalarTypeInfoFactory):
TypeName = Interface.DerivedProperty("int32")
CppType = Interface.DerivedProperty("std::int32_t")
CType = Interface.DerivedProperty("int32_t")
class BoolTypeInfoFactory(_ScalarTypeInfoFactory):
TypeName = Interface.DerivedProperty("bool")
CppType = Interface.DerivedProperty("bool")
CType = Interface.DerivedProperty("bool")
class VectorTypeInfoFactory(TypeInfoFactory):
# ----------------------------------------------------------------------
# |
# | Public Types
# |
# ----------------------------------------------------------------------
TypeName = Interface.DerivedProperty(
re.compile(r"vector\<(?P<type>\S+)\>"))
CppType = Interface.DerivedProperty(None)
# ----------------------------------------------------------------------
# |
# | Public Types
# |
# ----------------------------------------------------------------------
def __init__(
self,
custom_structs=None,
custom_enums=None,
member_type=None,
create_type_info_factory_func=None,
):
if member_type is not None:
assert create_type_info_factory_func is not None
match = self.TypeName.match(member_type)
assert match, member_type
the_type = match.group("type")
type_info = create_type_info_factory_func(the_type)
if not hasattr(type_info, "CType"):
raise Exception(
"'{}' is a type that can't be directly expressed in C and therefore cannot be used with a vector"
.format(the_type))
self._type_info = type_info
# Override the CppType property with this type info
self.CppType = "std::tuple<{type} *, {type} *>".format(
type=self._type_info.CppType, )
# ----------------------------------------------------------------------
@Interface.override
def GetInputInfo(self, arg_name, is_optional, invocation_template):
if is_optional:
validation_statements = textwrap.dedent(
"""\
if({name}_ptr == nullptr && {name}_elements != 0) throw std::invalid_argument("'{name}_elements' is not 0");
if({name}_ptr != nullptr && {name}_elements == 0) throw std::invalid_argument("'{name}_elements' is 0");
""", ).format(name=arg_name, )
invocation_statements = invocation_template.format(
"{name}_ptr ? std::make_tuple(const_cast<{type} *>({name}_ptr), const_cast<{type} *>({name}_ptr) + {name}_elements) : nonstd::optional<std::tuple<{type} const *, {type} const *>>()"
.format(
type=self._type_info.CType,
name=arg_name,
), )
else:
validation_statements = textwrap.dedent(
"""\
if({name}_ptr == nullptr) throw std::invalid_argument("'{name}_ptr' is null");
if({name}_elements == 0) throw std::invalid_argument("'{name}_elements' is 0");
""", ).format(name=arg_name, )
invocation_statements = invocation_template.format(
"std::make_tuple(const_cast<{type} *>({name}_ptr), const_cast<{type} *>({name}_ptr) + {name}_elements)"
.format(
type=self._type_info.CType,
name=arg_name,
), )
return self.Result(
[
"/*in*/ {type} const *{name}_ptr".format(
type=self._type_info.CType,
name=arg_name,
),
"/*in*/ size_t {}_elements".format(arg_name),
],
validation_statements,
invocation_statements,
)
# ----------------------------------------------------------------------
@Interface.override
def GetInputBufferInfo(self, arg_name, is_optional, invocation_template):
if is_optional:
raise Exception(
"Optional vector values are not supported for input buffers")
return self.Result(
[
"/*in*/ {type} const **{name}_values_ptr".format(
type=self._type_info.CType,
name=arg_name,
),
"/*in*/ size_t const *{}_sizes_ptr".format(arg_name),
"/*in*/ size_t {}_elements".format(arg_name),
],
textwrap.dedent(
"""\
if({name}_values_ptr == nullptr) throw std::invalid_argument("'{name}_values_ptr' is null");
if({name}_sizes_ptr == nullptr) throw std::invalid_argument("'{name}_sizes_ptr' is null");
if({name}_elements == 0) throw std::invalid_argument("'{name}_elements' is 0");
""", ).format(name=arg_name, ),
textwrap.dedent(
"""\
std::vector<std::tuple<{type} *, {type} *>> {name}_buffer;
{name}_buffer.reserve({name}_elements);
{type} const * const * const {name}_values_end_ptr({name}_values_ptr + {name}_elements);
while({name}_values_ptr != {name}_values_end_ptr) {{
{name}_buffer.emplace_back(const_cast<{type} *>(*{name}_values_ptr), const_cast<{type} *>(*{name}_values_ptr) + *{name}_sizes_ptr);
++{name}_values_ptr;
++{name}_sizes_ptr;
}}
{statement}
""", ).format(
type=self._type_info.CType,
name=arg_name,
statement=invocation_template.format(
"{name}_buffer.data(), {name}_buffer.size()".format(
name=arg_name, ), ),
),
)
# ----------------------------------------------------------------------
@Interface.override
def GetOutputInfo(
self,
arg_name,
result_name="result",
is_struct_member=False,
):
return self.Result(
[
"/*out*/ {type} **{name}_ptr".format(
type=self._type_info.CType,
name=arg_name,
),
"/*out*/ size_t *{}_elements".format(arg_name),
],
textwrap.dedent(
"""\
if({name}_ptr == nullptr) throw std::invalid_argument("'{name}_ptr' is null");
if({name}_elements == nullptr) throw std::invalid_argument("'{name}_elements' is null");
""", ).format(name=arg_name, ),
# TODO: This code is assuming a result type of std::vector which
# will likely change in the future. This code will have to be
# updated when that change is made.
textwrap.dedent(
"""\
if({result_name}.empty()) {{
{pointer}{name}_ptr = nullptr;
{pointer}{name}_elements = 0;
}}
else {{
struct {name}Internal {{
static void Deleter({type} *pData) {{
delete [] pData;
}}
}};
std::unique_ptr<{type}, void (*)({type} *)> pBuffer(new {type}[{result_name}.size()], {name}Internal::Deleter);
{type} * ptr(pBuffer.get());
for(auto &value : {result_name})
*ptr++ = std::move(value);
{pointer}{name}_ptr = pBuffer.release();
{pointer}{name}_elements = {result_name}.size();
}}
""", ).format(
type=self._type_info.CType,
name=arg_name,
result_name=result_name,
pointer="" if is_struct_member else "*",
),
)
# ----------------------------------------------------------------------
@Interface.override
def GetDestroyOutputInfo(
self,
arg_name="result",
):
return self.Result(
[
"/*in*/ {type} *{name}_ptr".format(
type=self._type_info.CType,
name=arg_name,
),
"/*in*/ size_t {}_elements".format(arg_name),
],
textwrap.dedent(
"""\
if({name}_ptr == nullptr && {name}_elements != 0) throw std::invalid_argument("Invalid buffer");
if({name}_ptr != nullptr && {name}_elements == 0) throw std::invalid_argument("Invalid buffer");
""", ).format(name=arg_name, ),
textwrap.dedent(
"""\
if({name}_ptr)
delete [] {name}_ptr;
""", ).format(name=arg_name, ),
)
