class TimePointTypeInfo(_StructTypeInfo):
TypeName = Interface.DerivedProperty("TimePoint")
CppType = Interface.DerivedProperty("TimePoint")
class FloatTypeInfoFactory(_ScalarTypeInfoFactory):
TypeName = Interface.DerivedProperty("float")
CppType = Interface.DerivedProperty("std::float_t")
CType = Interface.DerivedProperty("float")
class BoolTypeInfoFactory(_ScalarTypeInfoFactory):
TypeName = Interface.DerivedProperty("bool")
CppType = Interface.DerivedProperty("bool")
CType = Interface.DerivedProperty("bool")
class SingleValueSparseVectorTypeInfo(TypeInfo):
# ----------------------------------------------------------------------
# |
# | Public Types
# |
# ----------------------------------------------------------------------
TypeName = Interface.DerivedProperty(
re.compile(r"single_value_sparse_vector\<(?P<type>\S+)\>"))
CppType = Interface.DerivedProperty(None)
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
def __init__(self,
*args,
member_type=None,
create_type_info_func=None,
**kwargs):
if member_type is None:
return
assert create_type_info_func is not None
super(SingleValueSparseVectorTypeInfo, self).__init__(*args, **kwargs)
match = self.TypeName.match(member_type)
assert match, member_type
the_type = match.group("type")
type_info = create_type_info_func(the_type)
if not hasattr(type_info, "CType"):
raise Exception(
"'{}' is a type that can't be directly expressed in C and therefore cannot be used with a single_value_sparse_vector"
.format(the_type))
if type_info.IsOptional:
raise Exception(
"SingleValueSparseVector types do not currently support optional values ('{}')"
.format(the_type))
self._type_info = type_info
# ----------------------------------------------------------------------
@Interface.override
def GetTransformInputArgs(
self,
input_name="input",
):
raise NotImplementedError("This structure is only used during output")
# ----------------------------------------------------------------------
@Interface.override
def GetTransformInputBufferArgs(
self,
input_name='input',
):
raise NotImplementedError("This structure is only used during output")
# ----------------------------------------------------------------------
@Interface.override
def GetOutputInfo(
self,
invocation_template,
result_name="result",
):
return self.Result(
"Microsoft::Featurizer::Featurizers::SingleValueSparseVectorEncoding<{}>"
.format(self._type_info.CppType),
[
self.Type("uint64_t", "{}_numElements".format(result_name)),
self.Type(self._type_info.CppType,
"{}_value".format(result_name)),
self.Type("uint64_t", "{}_index".format(result_name)),
],
invocation_template.format(
"{result}_numElements, {result}_value, {result}_index".format(
result=result_name, ), ),
None,
)
class Plugin(PythonSerializationImpl):
# ----------------------------------------------------------------------
# | Properties
Name = Interface.DerivedProperty("PythonJson")
Description = Interface.DerivedProperty(
"Creates python code that is able to serialize and deserialize python objects to JSON"
)
# ----------------------------------------------------------------------
# | Methods
@classmethod
@Interface.override
def GetAdditionalGeneratorItems(cls, context):
return [
_script_fullpath, PythonDestinationStatementWriter,
PythonSourceStatementWriter
] + super(Plugin, cls).GetAdditionalGeneratorItems(context)
# ----------------------------------------------------------------------
# | Private Types
@Interface.staticderived
class SourceStatementWriter(PythonSourceStatementWriter):
# ----------------------------------------------------------------------
# | Public Properties
ObjectTypeDesc = Interface.DerivedProperty("a JSON object")
# ----------------------------------------------------------------------
# | Methods
@classmethod
@Interface.override
def ConvenienceConversions(cls, var_name, element_or_none):
content = textwrap.dedent(
"""\
if isinstance({var_name}, six.string_types):
if FileSystem.IsFilename({var_name}):
with open({var_name}) as f:
{var_name} = json.load(f)
else:
{var_name} = json.loads({var_name})
""", ).format(var_name=var_name, )
if element_or_none is not None:
content += textwrap.dedent(
"""\
{}
""", ).format(
super(Plugin.SourceStatementWriter,
cls).ConvenienceConversions(
var_name, element_or_none))
return content
# ----------------------------------------------------------------------
@Interface.staticderived
class DestinationStatementWriter(PythonDestinationStatementWriter):
# ----------------------------------------------------------------------
# | Public Properties
ObjectTypeDesc = Interface.DerivedProperty("a JSON object")
# ----------------------------------------------------------------------
# | Methods
@staticmethod
@Interface.override
def SerializeToString(var_name):
return "_JsonToString({var_name}, pretty_print)".format(
var_name=var_name, )
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetGlobalUtilityMethods(source_writer):
return textwrap.dedent(
"""\
# ----------------------------------------------------------------------
def _JsonToString(obj, pretty_print):
if pretty_print:
content = json.dumps(obj, cls=JsonEncoder, indent=2, separators=[", ", " : "])
# Remove trailing whitespace
return "\\n".join([line.rstrip() for line in content.split("\\n")])
else:
return json.dumps(obj, cls=JsonEncoder)
""", )
# ----------------------------------------------------------------------
# | Private Properties
_SupportAttributes = Interface.DerivedProperty(False)
_SupportAnyElements = Interface.DerivedProperty(True)
_SupportDictionaryElements = Interface.DerivedProperty(True)
_TypeInfoSerializationName = Interface.DerivedProperty("JsonSerialization")
_SourceStatementWriter = Interface.DerivedProperty(SourceStatementWriter)
_DestinationStatementWriter = Interface.DerivedProperty(
DestinationStatementWriter)
# ----------------------------------------------------------------------
# | Private Methods
@staticmethod
@Interface.override
def _WriteFileHeader(output_stream):
output_stream.write(
textwrap.dedent(
"""\
import json
from CommonEnvironment import FileSystem
from CommonEnvironment.TypeInfo.FundamentalTypes.Serialization.JsonSerialization import JsonSerialization
# ----------------------------------------------------------------------
class JsonEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, Object):
d = copy.deepcopy(o.__dict__)
for k in list(six.iterkeys(d)):
if k.startswith("_"):
del d[k]
return d
return getattr(o, "__dict__", o)
""", ), )
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def _WriteFileFooter(output_stream):
# Nothing to do here
pass
class UInt32TypeInfo(_ScalarTypeInfo):
TypeName = Interface.DerivedProperty("uint32")
CType = Interface.DerivedProperty("uint32_t")
CppType = Interface.DerivedProperty("std::uint32_t")
def _Impl( display_sentinel,
json_filename,
result_filename,
first,
output_stream,
method_name,
parser,
):
output_stream = StreamDecorator( output_stream,
line_prefix=display_sentinel,
)
with open(json_filename) as f:
try:
data = parser(f.read(), is_root=True)
except Exception as ex:
output_stream.write("ERROR: {} ({})\n".format(str(ex), ex.stack))
return -1
output_stream.write("Parsing dependencies...")
with output_stream.DoneManager():
dependencies = ActivationData.Load(None, None, None).PrioritizedRepositories
has_config_specific = False
output_stream.write("Validating...")
with output_stream.DoneManager() as dm:
for index, repository_info in enumerate(dependencies):
dm.stream.write("Processing '{}' ({} of {})...".format( repository_info.Name,
index + 1,
len(dependencies),
))
with dm.stream.DoneManager() as this_dm:
with Utilities.CustomMethodManager(os.path.join(repository_info.Root, Constants.HOOK_ENVIRONMENT_CUSTOMIZATION_FILENAME), method_name) as method:
if not method:
continue
args = OrderedDict([ ( "data", data ),
( "output_stream", this_dm.stream ),
])
# Get the method args to see if a configuration is requried
func_code = six.get_function_code(method)
if "configuration" in func_code.co_varnames[:func_code.co_argcount]:
args["configuration"] = repository_info.Configuration
has_config_specific = True
elif not first:
# Don't call a config-agnostic method more than once
continue
try:
this_dm.result = Interface.CreateCulledCallable(method)(args) or 0
except Exception as ex:
this_dm.stream.write(StringHelpers.LeftJustify( "ERROR: {}\n".format(str(ex).rstrip()),
len("ERROR: "),
))
this_dm.result = -1
with open(result_filename, 'w') as f:
f.write('-1' if dm.result != 0 else '1' if has_config_specific else '0')
return dm.result
class DoubleTypeInfo(_FloatingPointTypeInfo):
TypeName = Interface.DerivedProperty("double")
CSharpType = Interface.DerivedProperty("double")
CSharpTypeName = Interface.DerivedProperty("Double")
class BoolTypeInfo(_ScalarTypeInfo):
TypeName = Interface.DerivedProperty("bool")
CSharpType = Interface.DerivedProperty("bool")
CSharpTypeName = Interface.DerivedProperty("Bool")
class UInt64TypeInfo(_ScalarTypeInfo):
TypeName = Interface.DerivedProperty("uint64")
CSharpType = Interface.DerivedProperty("ulong")
CSharpTypeName = Interface.DerivedProperty("UInt64")
class FloatTypeInfo(_FloatingPointTypeInfo):
TypeName = Interface.DerivedProperty("float")
CSharpType = Interface.DerivedProperty("float")
CSharpTypeName = Interface.DerivedProperty("Float")
class UInt32TypeInfo(_ScalarTypeInfo):
TypeName = Interface.DerivedProperty("uint32")
CSharpType = Interface.DerivedProperty("uint")
CSharpTypeName = Interface.DerivedProperty("UInt32")
class UInt8TypeInfo(_ScalarTypeInfo):
TypeName = Interface.DerivedProperty("uint8")
CSharpType = Interface.DerivedProperty("byte")
CSharpTypeName = Interface.DerivedProperty("UInt8")
class PythonSourceStatementWriter(SourceStatementWriter):
ObjectTypeDesc = Interface.DerivedProperty("a python object")
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def ConvenienceConversions(cls, var_name, element_or_none):
if element_or_none is None:
return ""
return textwrap.dedent(
"""\
if not isinstance({var_name}, list):
if isinstance({var_name}, dict) and "{name}" in {var_name}:
{var_name} = {var_name}["{name}"]
elif not isinstance({var_name}, dict) and hasattr({var_name}, "{name}"):
{var_name} = getattr({var_name}, "{name}")
elif is_root:
{var_name} = DoesNotExist
""", ).format(
var_name=var_name,
name=element_or_none.Name,
)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetChild(
cls,
var_name,
child_element,
is_simple_schema_fundamental=False,
):
if is_simple_schema_fundamental:
is_optional = False
else:
is_optional = child_element.TypeInfo.Arity.Min == 0
return textwrap.dedent(
"""\
cls._GetPythonAttribute(
{var_name},
{name},
is_optional={is_optional},
)
""", ).format(
var_name=var_name,
name=cls.GetElementStatementName(child_element),
is_optional=is_optional,
)
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetFundamental(var_name, child_element):
return var_name
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetAdditionalDataChildren():
return '[(k, v) for k, v in six.iteritems(source if isinstance(source, dict) else getattr(source, "__dict__", {})) if not k.startswith("_") and k not in exclude_names]'
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def CreateAdditionalDataItem(cls, dest_writer, name_var_name,
source_var_name):
temporary_element = cls.CreateTemporaryElement(name_var_name, "1")
temporary_children_element = cls.CreateTemporaryElement("k", "+")
return textwrap.dedent(
"""\
# The following types should be returned directly without additional conversion
if isinstance({source_var_name}, (int, float, str, bool)):
return {source_var_name}
assert not isinstance({source_var_name}, list), {source_var_name}
if not isinstance({source_var_name}, dict):
{source_var_name} = {source_var_name}.__dict__
source_attribute_names = {source_var_name}.get("{attribute_names}", set())
attributes = OrderedDict()
items = OrderedDict()
for k, v in six.iteritems(source):
if k.startswith("_"):
continue
if k in source_attribute_names:
attributes[k] = v
else:
items[k] = v
if len(items) == 1 and next(six.iterkeys(items)) == {source_var_name}.get("{fundamental_name}", None):
return {simple_statement}
result = {compound_statement}
for k, v in six.iteritems(items):
try:
if isinstance(v, list):
new_items = []
for index, child in enumerate(v):
try:
new_items.append(cls._CreateAdditionalDataItem("item", child))
except:
_DecorateActiveException("Index {{}}".format(index))
{append_children}
else:
new_item = cls._CreateAdditionalDataItem(k, v)
{append_child}
except:
_DecorateActiveException(k)
return result
""", ).format(
source_var_name=source_var_name,
attribute_names=cls.ATTRIBUTES_ATTRIBUTE_NAME,
fundamental_name=cls.SIMPLE_ELEMENT_FUNDAMENTAL_ATTRIBUTE_NAME,
simple_statement=StringHelpers.LeftJustify(
dest_writer.CreateSimpleElement(
temporary_element,
"attributes",
'{}[{}["{}"]]'.format(
source_var_name, source_var_name,
cls.SIMPLE_ELEMENT_FUNDAMENTAL_ATTRIBUTE_NAME),
),
4,
).strip(),
compound_statement=dest_writer.CreateCompoundElement(
temporary_element, "attributes").strip(),
append_children=StringHelpers.LeftJustify(
dest_writer.AppendChild(
temporary_children_element, "result",
dest_writer.CreateCollection(
temporary_children_element, "new_items")),
12,
).strip(),
append_child=StringHelpers.LeftJustify(
dest_writer.AppendChild(
cls.CreateTemporaryElement("k", "1"), "result",
"new_item"), 8).strip(),
)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetGlobalUtilityMethods(cls, dest_writer):
return PythonStatementWriterMixin.GetGlobalUtilityMethods(
cls.ATTRIBUTES_ATTRIBUTE_NAME)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetClassUtilityMethods(cls, dest_writer):
return textwrap.dedent(
"""\
# ----------------------------------------------------------------------
@staticmethod
def _GetPythonAttribute(
item,
attribute_name,
is_optional=False,
):
if not isinstance(item, dict):
if hasattr(item, "__dict__"):
item = item.__dict__
else:
item = {}
value = item.get(attribute_name, DoesNotExist)
if value is DoesNotExist and not is_optional:
raise SerializeException("No items were found")
return value
""", )
class Plugin(PluginBase):
# ----------------------------------------------------------------------
# | Public Properties
Name = Interface.DerivedProperty("JsonSchema")
Description = Interface.DerivedProperty(
"Generates a JSON Schema file (https://json-schema.org/)")
Flags = Interface.DerivedProperty(
# ParseFlag.SupportAttributes
ParseFlag.SupportIncludeStatements
# | ParseFlag.SupportConfigStatements
# | ParseFlag.SupportExtensionsStatements
# | ParseFlag.SupportUnnamedDeclarations
# | ParseFlag.SupportUnnamedObjects
| ParseFlag.SupportNamedDeclarations
| ParseFlag.SupportNamedObjects
| ParseFlag.SupportRootDeclarations
| ParseFlag.SupportRootObjects
| ParseFlag.SupportChildDeclarations
| ParseFlag.SupportChildObjects
# | ParseFlag.SupportCustomElements
| ParseFlag.SupportAnyElements
| ParseFlag.SupportReferenceElements
| ParseFlag.SupportListElements
# | ParseFlag.SupportSimpleObjectElements
| ParseFlag.SupportVariantElements, )
# ----------------------------------------------------------------------
# | Methods
@staticmethod
@Interface.override
def IsValidEnvironment():
return True
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GenerateCustomSettingsAndDefaults():
yield "id", None
yield "description", None
yield "schema_version", "http://json-schema.org/draft-07/schema#"
yield "process_additional_data", False
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GenerateOutputFilenames(cls, context):
return ["{}.schema.json".format(context["output_name"])]
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetOptionalMetadataItems(cls, item):
results = []
if item.element_type == Elements.CompoundElement:
results.append(
Attributes.Attribute(
"process_additional_data",
BoolTypeInfo(arity="?", ),
default_value=None,
), )
return results + super(Plugin, cls).GetOptionalMetadataItems(item)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def Generate(
cls,
simple_schema_generator,
invoke_reason,
input_filenames,
output_filenames,
name,
elements,
include_indexes,
status_stream,
verbose_stream,
verbose,
id,
description,
schema_version,
process_additional_data,
):
assert len(output_filenames) == 1
output_filename = output_filenames[0]
del output_filenames
include_map = cls._GenerateIncludeMap(elements, include_indexes)
include_dotted_names = set(six.iterkeys(include_map))
top_level_elements = [
element for element in elements
if element.Parent is None and not element.IsDefinitionOnly
and element.DottedName in include_map
]
# ----------------------------------------------------------------------
def CreateDefinitions():
definitions_schema = {}
# ----------------------------------------------------------------------
class Visitor(Elements.ElementVisitor):
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def OnExitingElement(element):
if not isinstance(element, Elements.ReferenceElement):
definitions_schema["_{}".format(
element.DottedName)] = cls._Collectionize(
element, {
"$ref":
"#/definitions/_{}_Item".format(
element.DottedName)
})
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def OnFundamental(element):
definitions_schema["_{}_Item".format(
element.DottedName
)] = _FundamentalTypeInfoVisitor.Accept(element.TypeInfo)
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def OnCompound(element):
properties = OrderedDict()
required = []
for child in cls._EnumerateChildren(
element,
include_definitions=False,
):
properties[child.Name] = {
"$ref":
"#/definitions/_{}".format(
child.Resolve().DottedName)
}
if child.TypeInfo.Arity.Min != 0:
required.append(child.Name)
if isinstance(element,
Elements.CompoundElement) and hasattr(
element, "FundamentalAttributeName"):
properties[element.FundamentalAttributeName] = {
"$ref":
"#/definitions/_{}.{}".format(
element.DottedName,
element.FundamentalAttributeName)
}
required.append(element.FundamentalAttributeName)
definitions_schema["_{}.{}".format(
element.DottedName,
element.FundamentalAttributeName
)] = _FundamentalTypeInfoVisitor.Accept(
element.TypeInfo.Items[
element.FundamentalAttributeName], )
schema = {"type": "object", "properties": properties}
if required:
required.sort()
schema["required"] = required
element_process_additional_data = getattr(
element, "process_additional_data", None)
if element_process_additional_data is None:
element_process_additional_data = process_additional_data
if not element_process_additional_data:
schema["additionalProperties"] = False
definitions_schema["_{}_Item".format(
element.DottedName)] = schema
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def OnSimple(element):
raise Exception("SimpleElements are not supported")
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def OnVariant(this_cls, element):
any_of_options = []
for variation in element.Variations:
assert variation.TypeInfo.Arity.IsSingle
if not isinstance(variation,
Elements.ReferenceElement):
assert isinstance(
variation,
Elements.FundamentalElement), variation
definitions_schema["_{}_Item".format(
variation.DottedName
)] = _FundamentalTypeInfoVisitor.Accept(
variation.TypeInfo)
any_of_options.append({
"$ref":
"#/definitions/_{}_Item".format(
variation.Resolve().DottedName)
})
definitions_schema["_{}_Item".format(
element.DottedName)] = {
"anyOf": any_of_options
}
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def OnReference(element):
# References don't need to be added, as they will be resolved inline.
pass
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def OnList(element):
definitions_schema["_{}_Item".format(
element.DottedName)] = {
"$ref":
"#/definitions/_{}".format(
element.Reference.Resolve().DottedName)
}
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def OnAny(element):
definitions_schema["_{}_Item".format(
element.DottedName)] = {} # Empty schema
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def OnCustom(element):
raise Exception("CustomElements are not supported")
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def OnExtension(element):
raise Exception("ExtensionElements are not supported")
# ----------------------------------------------------------------------
Visitor().Accept(
elements,
include_dotted_names=include_dotted_names,
)
return definitions_schema
# ----------------------------------------------------------------------
def CreateElements():
schema = {}
for element in top_level_elements:
schema[element.DottedName] = {
"$ref":
"#/definitions/_{}".format(element.Resolve().DottedName)
}
return schema
# ----------------------------------------------------------------------
status_stream.write("Creating '{}'...".format(output_filename))
with status_stream.DoneManager() as dm:
schema = {
"$schema": schema_version,
"type": "object",
"definitions": CreateDefinitions()
}
if len(top_level_elements) > 1:
schema["properties"] = CreateElements()
required = []
for element in elements:
if element.DottedName in include_dotted_names and not element.IsDefinitionOnly and element.TypeInfo.Arity.Min != 0:
required.append(element.Name)
if required:
required.sort()
schema["required"] = required
if not process_additional_data:
schema["additionalProperties"] = False
elif top_level_elements:
schema["$ref"] = "#/definitions/_{}".format(
top_level_elements[0].DottedName)
if id:
schema["id"] = id
if description:
schema["description"] = description
with open(output_filename, "w") as f:
json.dump(
schema,
f,
indent=2,
separators=[", ", " : "],
sort_keys=True,
)
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
@staticmethod
def _Collectionize(element, schema):
arity = element.TypeInfo.Arity
if arity.Max == 1:
if arity.Min == 0 and hasattr(element, "default"):
schema["default"] = StringSerialization.DeserializeItem(
element.TypeInfo, element.default)
return schema
schema = {"type": "array", "items": schema}
if arity.Min != 0:
schema["minItems"] = arity.Min
if arity.Max is not None:
schema["maxItems"] = arity.Max
return schema
class Int8TypeInfo(_ScalarTypeInfo):
TypeName = Interface.DerivedProperty("int8")
CSharpType = Interface.DerivedProperty("sbyte")
CSharpTypeName = Interface.DerivedProperty("Int8")
class Int64TypeInfo(_ScalarTypeInfo):
TypeName = Interface.DerivedProperty("int64")
CType = Interface.DerivedProperty("int64_t")
CppType = Interface.DerivedProperty("std::int64_t")
class Int16TypeInfo(_ScalarTypeInfo):
TypeName = Interface.DerivedProperty("int16")
CSharpType = Interface.DerivedProperty("short")
CSharpTypeName = Interface.DerivedProperty("Int16")
class RelationalPluginImpl(PluginBase):
# ----------------------------------------------------------------------
# |
# | Public Properties
# |
# ----------------------------------------------------------------------
Flags = Interface.DerivedProperty(
0
# | ParseFlag.SupportAttributes
| ParseFlag.SupportIncludeStatements
# | ParseFlag.SupportConfigStatements
| ParseFlag.SupportExtensionsStatements
# | ParseFlag.SupportUnnamedDeclarations
# | ParseFlag.SupportUnnamedObjects
| ParseFlag.SupportNamedDeclarations
| ParseFlag.SupportNamedObjects
| ParseFlag.SupportRootDeclarations
| ParseFlag.SupportRootObjects
| ParseFlag.SupportChildDeclarations
| ParseFlag.SupportChildObjects
# | ParseFlag.SupportCustomElements
# | ParseFlag.SupportAnyElements
| ParseFlag.SupportReferenceElements
| ParseFlag.SupportListElements
# | ParseFlag.SupportSimpleObjectElements
# | ParseFlag.SupportVariantElements
# | ParseFlag.SupportDictionaryElements
| ParseFlag.MaintainAugmentingReferences
| ParseFlag.MaintainReferenceArity)
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def IsValidEnvironment():
return True
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GenerateCustomSettingsAndDefaults():
return []
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetExtensions():
return [
# constraint("SQL statement")
Extension(
"constraint",
allow_duplicates=True,
),
]
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetRequiredMetadataItems(item):
if (item.element_type == CompoundElement
and item.ItemType == type(item).ItemType.Standard):
return [
Attribute(
"identity",
EnumTypeInfo([e.name for e in Object.IdentityType]),
is_metadata=True,
),
]
return []
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def GetOptionalMetadataItems(item):
if item.element_type == FundamentalElement:
return [
Attribute(
"mutable",
BoolTypeInfo(),
default_value=False,
is_metadata=True,
),
Attribute(
"index",
BoolTypeInfo(),
default_value=False,
is_metadata=True,
),
Attribute(
"unique",
BoolTypeInfo(),
default_value=False,
is_metadata=True,
),
]
if item.element_type in [ListElement, ReferenceElement]:
assert len(item.references) == 1, item.references
reference = item.references[0]
if reference.element_type == CompoundElement:
return [
Attribute(
"mutable",
BoolTypeInfo(),
default_value=False,
is_metadata=True,
),
Attribute(
"backref",
BoolTypeInfo(),
is_metadata=True,
),
Attribute(
"backref_name",
StringTypeInfo(),
is_metadata=True,
validate_func=_ValidateBackrefName,
),
Attribute(
"backref_is_one_to_one",
BoolTypeInfo(),
is_metadata=True,
validate_func=_ValidateBackrefIsOneToOne,
),
]
return []
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def PreprocessContext(cls, context):
# Augment all elements with their corresponding relational elements
all_objects = []
all_elements = pickle.loads(context["pickled_elements"])
# Pass 1: Create objects
for element in _EnumCompoundElements(all_elements):
element.Object = Object.FromElement(element)
all_objects.append(element.Object)
# Pass 2: Relationships
for element in _EnumCompoundElements(all_elements):
if element.Parent:
element.Object.Add(Relationship.FromChild(element))
for child in cls._EnumerateChildren(
element,
include_definitions=False,
):
if isinstance(child, FundamentalElement):
element.Object.Add(
Fundamental(
child,
StringHelpers.ToSnakeCase(child.Name),
child.TypeInfo,
is_identity=False,
is_mutable=child.mutable,
is_index=child.index,
is_unique=child.unique,
), )
elif isinstance(child, CompoundElement):
# Nothing to do here, as the child will create a reference to
# this element when created.
pass
elif isinstance(child, ExtensionElement):
content = child.PositionalArguments[0]
if content.startswith('"'): content = content[1:]
if content.endswith('"'): content = content[:-1]
element.Object.Add(content)
elif isinstance(child, ReferenceElement):
resolved_element = child.Resolve()
if isinstance(resolved_element, FundamentalElement):
element.Object.Add(
Fundamental(
child,
StringHelpers.ToSnakeCase(child.Name),
child.TypeInfo,
is_identity=False,
is_mutable=getattr(child, "mutable", False),
is_index=getattr(child, "index", False),
is_unique=getattr(child, "unique", False),
), )
elif isinstance(resolved_element, CompoundElement):
element.Object.Add(
Relationship.FromReference(child,
resolved_element))
else:
assert False, resolved_element
elif isinstance(child, ListElement):
assert isinstance(child.Reference,
CompoundElement), child.Reference
element.Object.Add(
Relationship.FromReference(child, child.Reference))
else:
assert False, child
cls.AllElements = all_elements
cls.AllObjects = all_objects
return context
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def PostprocessContext(cls, context):
# The following validation methods validate the relationship between elements where
# as the validation associated with attributes only look at the associated element
# in isolation.
# ----------------------------------------------------------------------
def ValidateDefinitionCompoundElementChildren(element):
for child in cls._EnumerateChildren(
element,
include_standard=True,
include_definitions=True,
):
if not isinstance(
child, FundamentalElement) or child.IsDefinitionOnly:
return {
"child_source": child.Source,
"child_line": child.Line,
"child_column": child.Column,
}
return True
# ----------------------------------------------------------------------
def ValidateIntegerIdentityRelationships(element):
for child in cls._EnumerateChildren(
element,
include_standard=True,
include_definitions=False,
):
if (isinstance(child, (ListElement, ReferenceElement))
and child.TypeInfo.Arity.Min == 1
and child.TypeInfo.Arity.Max != 1):
return False
return True
# ----------------------------------------------------------------------
validation_data = [
(
"Top-level elements must be CompoundElements",
lambda element: not element.Parent and not element.
IsDefinitionOnly,
lambda element: isinstance(element, CompoundElement),
),
(
"Elements must have an arity of 1, ?, +, or *",
lambda element: not isinstance(element, ExtensionElement),
lambda element:
(element.TypeInfo.Arity.IsSingle or element.TypeInfo.Arity.
IsOptional or element.TypeInfo.Arity.IsOneOrMore or element.
TypeInfo.Arity.IsZeroOrMore),
),
(
"CompoundElements must have an arity of *",
lambda element: isinstance(element, CompoundElement) and
not element.IsDefinitionOnly,
lambda element: element.TypeInfo.Arity.IsZeroOrMore,
),
(
"CompoundElements with many-to-many references may not have the 'integer' identity, as it isn't possible to insert into the M2M table when the referencing id is not yet known",
lambda element: isinstance(element, CompoundElement
) and not element.IsDefinitionOnly
and element.identity == Object.IdentityType.Integer,
ValidateIntegerIdentityRelationships,
),
(
"FundamentalElements must have an arity of 1 or ?",
lambda element: isinstance(element, FundamentalElement),
lambda element: element.TypeInfo.Arity.IsSingle or element.
TypeInfo.Arity.IsOptional,
),
(
"'Definition' CompoundElements must have an arity of 1",
lambda element: isinstance(element, CompoundElement) and
element.IsDefinitionOnly,
lambda element: element.TypeInfo.Arity.IsSingle,
),
(
"'Definition' CompoundElements may not be referenced directory", # They can only appear as the base of another class
lambda element: isinstance(element, ReferenceElement),
lambda element: not isinstance(element.Resolve(
), CompoundElement) and element.Resolve().IsDefinitionOnly,
),
(
"'constraint' elements must be associated with a CompoundElement",
lambda element: isinstance(element, ExtensionElement) and
element.Name == "constraint",
lambda element: bool(element.Parent),
),
(
"'constraint' elements may only have one positional argument",
lambda element: isinstance(element, ExtensionElement) and
element.Name == "constraint",
lambda element: len(element.PositionalArguments
) == 1 and not element.KeywordArguments,
),
(
"'ReferenceElements' may only reference Compound- or Fundamental-Elements",
lambda element: isinstance(element, ReferenceElement),
lambda element: isinstance(element.Resolve(), (
CompoundElement, FundamentalElement)),
),
(
"'ListElements' may only reference CompoundElements",
lambda element: isinstance(element, ListElement),
lambda element: isinstance(element.Reference.Resolve(),
CompoundElement),
),
(
"'Definition' CompoundElements may only contain fundamental elements ({child_source} <{child_line} [{child_column}]>)",
lambda element: isinstance(element, CompoundElement) and
element.IsDefinitionOnly,
ValidateDefinitionCompoundElementChildren,
),
]
for element in cls.AllElements:
for failure_message, applies_func, is_valid_func in validation_data:
if not applies_func(element):
continue
result = is_valid_func(element)
if result is True:
continue
if isinstance(result, dict):
failure_message = failure_message.format(**result)
raise SimpleSchemaException(
element.Source,
element.Line,
element.Column,
failure_message,
)
return context
# ----------------------------------------------------------------------
# |
# | Protected Data
# |
# ----------------------------------------------------------------------
AllElements = None # Set in PreprocessContext
AllObjects = None # Set in PreprocessContext
class TestParser(TestParserImpl):
"""Parses content produced by Cmake"""
# ----------------------------------------------------------------------
# | Public Properties
Name = Interface.DerivedProperty("CMake")
Description = Interface.DerivedProperty("Parses CMake CTest output.")
# ----------------------------------------------------------------------
# | Methods
@staticmethod
@Interface.override
def IsSupportedCompiler(compiler):
return compiler.Name == "CMake"
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def Parse(test_data):
if "100% tests passed" not in test_data:
return -1
# CTest will append an index before each line of the test output -
# remove that if it exists.
line_regex = re.compile(r"^\d+: (?P<content>.*)")
lines = test_data.split("\n")
for index, line in enumerate(lines):
match = line_regex.match(line)
if not match:
continue
lines[index] = match.group("content")
scrubbed_test_data = "\n".join(lines)
# CTest can wrap many individual test frameworks - attempt to extract benchmark
# data from well-know test frameworks.
benchmark_data = None
for extract_func in [ExtractCatch2BenchmarkOutput]:
benchmark_data = extract_func(scrubbed_test_data)
if benchmark_data:
return 0, benchmark_data
return 0
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def CreateInvokeCommandLine(context, debug_on_error):
is_profile_or_benchmark = context.get("is_profile",
False) or context.get(
"is_benchmark",
False,
)
return 'cd "{output_dir}" && ctest --verbose{parallel}'.format(
output_dir=context["output_dir"],
parallel="" if is_profile_or_benchmark else " --parallel",
)
# ----------------------------------------------------------------------
@staticmethod
@Interface.override
def RemoveTemporaryArtifacts(context):
for potential_dir in ["Testing"]:
potential_dir = os.path.join(context["output_dir"], potential_dir)
FileSystem.RemoveTree(potential_dir)
class SparseVectorTypeInfoFactory(TypeInfoFactory):
# ----------------------------------------------------------------------
# |
# | Public Types
# |
# ----------------------------------------------------------------------
TypeName = Interface.DerivedProperty(re.compile(r"sparse_vector\<(?P<type>\S+)\>"))
CppType = Interface.DerivedProperty(None)
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
def __init__(
self,
custom_structs=None,
custom_enums=None,
member_type=None,
create_type_info_factory_func=None,
):
if member_type is not None:
assert create_type_info_factory_func is not None
match = self.TypeName.match(member_type)
assert match, member_type
the_type = match.group("type")
type_info = create_type_info_factory_func(the_type)
if not hasattr(type_info, "CType"):
raise Exception("'{}' is a type that can't be directly expressed in C and therefore cannot be used with a sparse_vector".format(the_type))
self._type_info = type_info
# ----------------------------------------------------------------------
@Interface.override
def GetInputInfo(self, arg_name, is_optional, invocation_template):
raise NotImplementedError("This structure is only used during output")
# ----------------------------------------------------------------------
@Interface.override
def GetInputBufferInfo(self, arg_name, is_optional, invocation_template):
raise NotImplementedError("This structure is only used during output")
# ----------------------------------------------------------------------
@Interface.override
def GetOutputInfo(
self,
arg_name,
result_name="result",
is_struct_member=False,
):
return self.Result(
[
"/*out*/ uint64_t * {}_numElements".format(arg_name),
"/*out*/ uint64_t * {}_numValues".format(arg_name),
"/*out*/ {} **{}_values".format(self._type_info.CType, arg_name),
"/*out*/ uint64_t **{}_indexes".format(arg_name),
],
textwrap.dedent(
"""\
if({name}_numElements == nullptr) throw std::invalid_argument("'{name}_numElements' is null");
if({name}_numValues == nullptr) throw std::invalid_argument("'{name}_numValues' is null");
if({name}_values == nullptr) throw std::invalid_argument("'{name}_values' is null");
if({name}_indexes == nullptr) throw std::invalid_argument("'{name}_indexes' is null");
""",
).format(
name=arg_name,
),
textwrap.dedent(
"""\
std::unique_ptr<{type} []> pValues(new {type} [{result}.Values.size()]);
std::unique_ptr<uint64_t []> pIndexes(new uint64_t [{result}.Values.size()]);
{type} * pValue(pValues.get());
uint64_t * pIndex(pIndexes.get());
for(auto const & encoding : {result}.Values) {{
*pValue++ = encoding.Value;
*pIndex++ = encoding.Index;
}}
{pointer}{name}_numElements = {result}.NumElements;
{pointer}{name}_numValues = {result}.Values.size();
{pointer}{name}_values = pValues.release();
{pointer}{name}_indexes = pIndexes.release();
""",
).format(
name=arg_name,
result=result_name,
type=self._type_info.CppType,
pointer="" if is_struct_member else "*",
),
)
# ----------------------------------------------------------------------
@Interface.override
def GetDestroyOutputInfo(
self,
arg_name="result",
):
return self.Result(
[
"/*in*/ uint64_t {}_numElements".format(arg_name),
"/*in*/ uint64_t {}_numValues".format(arg_name),
"/*in*/ {} const * {}_values".format(self._type_info.CType, arg_name),
"/*in*/ uint64_t const * {}_indexes".format(arg_name),
],
textwrap.dedent(
"""\
if({name}_numElements == 0) throw std::invalid_argument("'{name}_numElements' is 0");
if({name}_numValues == 0) throw std::invalid_argument("'{name}_numValues' is 0");
if({name}_values == nullptr) throw std::invalid_argument("'{name}_values' is null");
if({name}_indexes == nullptr) throw std::invalid_argument("'{name}_indexes' is null");
""",
).format(
name=arg_name,
),
textwrap.dedent(
"""\
delete [] {name}_values;
delete [] {name}_indexes;
""",
).format(
name=arg_name,
),
)
class TupleTypeInfo(TypeInfo):
# ----------------------------------------------------------------------
# |
# | Public Types
# |
# ----------------------------------------------------------------------
TypeName = Interface.DerivedProperty(re.compile(r"tuple\<(?P<types>.+)\>"))
CppType = Interface.DerivedProperty(None)
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
def __init__(self,
*args,
member_type=None,
create_type_info_func=None,
**kwargs):
if member_type is None:
return
assert create_type_info_func is not None
super(TupleTypeInfo, self).__init__(*args, **kwargs)
match = self.TypeName.match(member_type)
assert match, member_type
type_infos = []
for type_ in GetTemplateArgs(match.group("types")):
type_info = create_type_info_func(type_)
assert type_info, type_
type_infos.append(type_info)
self._type_infos = type_infos
self._create_type_info_func = create_type_info_func
# Override the CppType property with this type info
self.CppType = "std::tuple<{}>".format(
", ".join([type_info.CppType for type_info in self._type_infos]), )
# ----------------------------------------------------------------------
@Interface.override
def GetTransformInputArgs(
self,
input_name="input",
):
if self.IsOptional:
raise NotImplementedError(
"Optional tuples are not supported at this time")
parameters = []
prefix_statements = []
for index, type_info in enumerate(self._type_infos):
result = type_info.GetTransformInputArgs("{}{}".format(
input_name, index))
these_prefix_statements = "auto const & {name}{index}(std::get<{index}>({name}));".format(
name=input_name,
index=index,
)
if isinstance(result, tuple):
these_prefix_statements += "\n\n{}\n".format(
result[1].rstrip())
result = result[0]
parameters.append(result)
prefix_statements.append(these_prefix_statements)
return (
", ".join(parameters),
"\n".join(prefix_statements),
)
# ----------------------------------------------------------------------
@Interface.override
def GetTransformInputBufferArgs(
self,
input_name='input',
):
raise NotImplementedError("Not implemented yet")
# ----------------------------------------------------------------------
@Interface.override
def GetOutputInfo(
self,
invocation_template,
result_name="result",
):
if self.IsOptional:
raise NotImplementedError(
"Optional tuples are not supported at this time")
tuple_elements = []
local_variables = []
append_result_statements = []
destroy_arguments = []
for index, type_info in enumerate(self._type_infos):
# This may ultimately need to use something like the SharedLibraryPlugin's _InvocationTemplate and
# _ExtractDecoratedInvocationStatements for those types with very complicated initialization semantics.
result = type_info.GetOutputInfo(
"{}",
result_name="{}{}".format(result_name, index),
)
tuple_elements.append(type_info.CppType)
local_variables += result.TransformVars
append_result_statements.append(result.AppendResultStatement)
if result.DestroyArgs:
destroy_arguments.append(result.DestroyArgs)
return self.Result(
"std::tuple<{}>".format(", ".join(tuple_elements)),
local_variables,
invocation_template.format(", ".join(append_result_statements)),
None if not destroy_arguments else ", ".join(destroy_arguments),
destroy_inline=True,
)
class VectorTypeInfo(TypeInfo):
# ----------------------------------------------------------------------
# |
# | Public Properties
# |
# ----------------------------------------------------------------------
TypeName = Interface.DerivedProperty(
re.compile(r"vector\<(?P<type>\S+)\>"))
CSharpType = Interface.DerivedProperty("TODO1")
CSharpTypeName = Interface.DerivedProperty("TODO2")
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
def __init__(self,
*args,
member_type=None,
create_type_info_func=None,
**kwargs):
if member_type is None:
return
assert create_type_info_func is not None
super(VectorTypeInfo, self).__init__(*args, **kwargs)
match = self.TypeName.match(member_type)
assert match, member_type
the_type = match.group("type")
type_info = create_type_info_func(the_type)
assert type_info, the_type
self._type_info = type_info
# ----------------------------------------------------------------------
@Interface.override
def GetNativeInputInfo(self):
return self.Result(
"TODO3: Parameter decl",
"TODO4: Validation statements",
"TODO5: Invocation statements",
"TODO6: Conversion end",
"TODO7: Delete transformed data",
)
# ----------------------------------------------------------------------
@Interface.override
def GetNativeOutputInfo(
self,
is_struct=False,
featurizer_name="",
):
return self.Result(
"TODO8: Parameter decl",
"TODO9: Validation statements",
"TODO10: Invocation statements",
"TODO11: Conversion end",
"TODO12: Delete transformed data",
)
class DateTimeTypeInfoFactory(TypeInfoFactory):
# ----------------------------------------------------------------------
# |
# | Public Types
# |
# ----------------------------------------------------------------------
TypeName = Interface.DerivedProperty("datetime")
CppType = Interface.DerivedProperty(
"std::chrono::system_clock::time_point")
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetInputInfo(cls, arg_name, is_optional, invocation_template):
if is_optional:
param_decorator = "const *"
invocation = invocation_template.format(
"{name} ? CreateDateTime({name}) : nonstd::optional<{cpp_type}>()"
.format(
name=arg_name,
cpp_type=cls.CppType,
), )
else:
param_decorator = ""
invocation = invocation_template.format(
"CreateDateTime({name})".format(name=arg_name, ), )
return cls.Result(
[
"/*in*/ DateTimeParameter {param_decorator}{name}".format(
name=arg_name,
param_decorator=param_decorator,
),
],
"", # No validation
invocation,
)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetInputBufferInfo(cls, arg_name, is_optional, invocation_template):
if is_optional:
param_decorator = "const *"
validation_suffix = textwrap.dedent(
"""\
std::vector<nonstd::optional<{cpp_type}>> {name}_buffer;
{name}_buffer.reserve({name}_items};
DateTimeParameter const * const * const {name}_end({name}_ptr + {name}_items);
while({name}_ptr != {name}_end) {{
#if (defined __apple_build_version__)
{name}_buffer.push_back(*{name}_ptr ? CreateDateTime(**{name}_ptr) : nonstd::optional<{cpp_type}>());
#else
{name}_buffer.emplace_back(*{name}_ptr ? CreateDateTime(**{name}_ptr) : nonstd::optional<{cpp_type}>());
#endif
++{name}_ptr;
}}
""", ).format(
name=arg_name,
cpp_type=cls.CppType,
)
else:
param_decorator = ""
validation_suffix = textwrap.dedent(
"""\
std::vector<{cpp_type}> {name}_buffer;
{name}_buffer.reserve({name}_items);
DateTimeParameter const * const {name}_end({name}_ptr + {name}_items);
while({name}_ptr != {name}_end) {{
#if (defined __apple_build_version__)
{name}_buffer.push_back(CreateDateTime(*{name}_ptr));
#else
{name}_buffer.emplace_back(CreateDateTime(*{name}_ptr));
#endif
++{name}_ptr;
}}
""", ).format(
name=arg_name,
cpp_type=cls.CppType,
)
return cls.Result(
[
"/*in*/ DateTimeParameter const * {param_decorator}{name}_ptr".
format(
name=arg_name,
param_decorator=param_decorator,
),
"/*in*/ std::size_t {name}_items".format(name=arg_name, ),
],
textwrap.dedent(
"""\
if({name}_ptr == nullptr) throw std::invalid_argument("'{name}_ptr' is null");
if({name}_items == 0) throw std::invalid_argument("'{name}_items' is 0");
{validation_suffix}
""", ).format(
name=arg_name,
validation_suffix="\n{}\n".format(validation_suffix, )
if validation_suffix else "",
),
invocation_template.format(
"{name}_buffer.data(), {name}_buffer.size()".format(
name=arg_name, ), ),
)
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetOutputInfo(
cls,
arg_name,
result_name="result",
is_struct_member=False,
):
raise NotImplemented("Not implemented yet")
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetDestroyOutputInfo(
cls,
arg_name="result",
):
raise NotImplemented("Not implemented yet")
class UInt64TypeInfoFactory(_ScalarTypeInfoFactory):
TypeName = Interface.DerivedProperty("uint64")
CppType = Interface.DerivedProperty("std::uint64_t")
CType = Interface.DerivedProperty("uint64_t")
class StandardCustomStructInfo(CustomStructInfo):
"""\
Standard implementation that doesn't transform the structure, but rather
writes each element to an output tensor.
"""
# ----------------------------------------------------------------------
# |
# | Public Properties
# |
# ----------------------------------------------------------------------
StructName = Interface.DerivedProperty("")
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
def __init__(self, custom_struct):
type_to_template_lookup = OrderedDict()
for member in custom_struct.members:
if member.type not in type_to_template_lookup:
type_to_template_lookup[member.type] = "OutputT{}".format(
len(type_to_template_lookup),
)
template_to_types = OrderedDict()
for k, v in six.iteritems(type_to_template_lookup):
template_to_types[v] = k
# Commit
self._custom_struct = custom_struct
self._type_to_template_lookup = type_to_template_lookup
self._template_to_types = template_to_types
# ----------------------------------------------------------------------
@Interface.override
def GetDefOutputStatementsConstraintsAndSuffix(self):
# ----------------------------------------------------------------------
def ToOrtTypeString(value):
return value.replace("std::", "").replace("_t", "")
# ----------------------------------------------------------------------
output_statements = []
for index, member in enumerate(self._custom_struct.members):
output_statements.append(
'.Output({index}, "{name}", "{desc}", "{type}")'.format(
index=index,
name=member.name,
desc=member.description or "No information available",
type=self._type_to_template_lookup[member.type],
),
)
return (
output_statements,
OrderedDict(
[
(k, [ToOrtTypeString(v)])
for k,
v in six.iteritems(self._template_to_types)
],
),
textwrap.dedent(
"""\
.TypeAndShapeInferenceFunction(
[](ONNX_NAMESPACE::InferenceContext& ctx) {{
const bool has_shape = hasInputShape(ctx, 1);
{statements}
}}
)
""",
).format(
statements=StringHelpers.LeftJustify(
"\n".join(
[
textwrap.dedent(
"""\
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_{type}, {index});
if(has_shape) {{
propagateShapeFromInputToOutput(ctx, 1, {index});
}}
""",
).format(
index=index,
type=ToOrtTypeString(member.type).upper(),
)
for index, member in enumerate(self._custom_struct.members)
],
),
6,
),
)
)
# ----------------------------------------------------------------------
@Interface.override
def GetKernelInitializeAssignAndPreprocessorStatements(self, transformer_name, input_transformation_statement):
initialize_statements_part1 = []
initialize_statements_part2 = []
assign_statements = ["auto result(transformer.execute({}));\n".format(input_transformation_statement)]
for index, member in enumerate(self._custom_struct.members):
initialize_statements_part1.append(
"Tensor* {name}_tensor(ctx->Output({index}, input_tensor->Shape()));".format(
name=member.name,
index=index,
),
)
initialize_statements_part2.append(
"{type}* {name}_data({name}_tensor->MutableData<{type}>());".format(
name=member.name,
type=member.type,
),
)
assign_statements.append(
"{name}_data[i] = std::move(result.{name});".format(
name=member.name,
),
)
return (
initialize_statements_part1 + [""] + initialize_statements_part2,
assign_statements,
)
class DoubleTypeInfoFactory(_ScalarTypeInfoFactory):
TypeName = Interface.DerivedProperty("double")
CppType = Interface.DerivedProperty("std::double_t")
CType = Interface.DerivedProperty("double")
class Plugin(RelationalPluginImpl):
# ----------------------------------------------------------------------
# |
# | Public Properties
# |
# ----------------------------------------------------------------------
Name = Interface.DerivedProperty("DebugRelational")
Description = Interface.DerivedProperty(
"Displays information associated with objects created by the base Relational plugin class"
)
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GetAdditionalGeneratorItems(cls, context):
return super(Plugin, cls).GetAdditionalGeneratorItems(context) + [
RelationalPluginImpl
]
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def GenerateOutputFilenames(
cls,
context,
all_objects=None,
):
return [
os.path.join(context["output_dir"],
"{}.txt".format(context["output_name"]))
]
# ----------------------------------------------------------------------
@classmethod
@Interface.override
def Generate(
cls,
simple_schema_generator,
invoke_reason,
input_filenames,
output_filenames,
name,
elements,
include_indexes,
status_stream,
verbose_stream,
verbose,
):
with open(output_filenames[0], "w") as f:
f.write(cls._GenerateFileHeader())
for obj in cls.AllObjects:
f.write(
textwrap.dedent(
"""\
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
Unique Name: {unique}
Singular Name: {singular}
Plural Name: {plural}
""", ).format(
unique=obj.UniqueName,
singular=obj.SingularName,
plural=obj.PluralName,
), )
for child in obj.children:
f.write(
textwrap.dedent(
"""\
# ----------------------------------------------------------------------
{}
{}
""", ).format(
child.Name,
child.Item,
), )
f.write("\n")
class Int32TypeInfoFactory(_ScalarTypeInfoFactory):
TypeName = Interface.DerivedProperty("int32")
CppType = Interface.DerivedProperty("std::int32_t")
CType = Interface.DerivedProperty("int32_t")
class DateTimeTypeInfo(TypeInfo):
# ----------------------------------------------------------------------
# |
# | Public Types
# |
# ----------------------------------------------------------------------
TypeName = Interface.DerivedProperty("datetime")
CppType = Interface.DerivedProperty(
"std::chrono::system_clock::time_point")
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
@Interface.override
def GetInputInfo(self, arg_name, invocation_template):
if self.IsOptional:
param_decorator = "const *"
invocation = invocation_template.format(
"{name} ? CreateDateTime({name}) : nonstd::optional<{cpp_type}>()"
.format(
name=arg_name,
cpp_type=self.CppType,
), )
else:
param_decorator = ""
invocation = invocation_template.format(
"CreateDateTime({name})".format(name=arg_name, ), )
return self.Result(
[
self.Type("DateTimeParameter {}".format(param_decorator),
arg_name),
],
"", # No validation
invocation,
)
# ----------------------------------------------------------------------
@Interface.override
def GetInputBufferInfo(
self,
arg_name,
invocation_template,
items_var_name=None,
):
parameters = [
# The type will be filled in below
self.Type(None, "{}_ptr".format(arg_name)),
]
if items_var_name is None:
items_var_name = "{}_items".format(arg_name)
parameters.append(self.Type("size_t", items_var_name))
if self.IsOptional:
param_decorator = "const *"
validation_suffix = textwrap.dedent(
"""\
std::vector<nonstd::optional<{cpp_type}>> {name}_buffer;
{name}_buffer.reserve({items_var_name}};
while({name}_buffer.size() < {items_var_name}) {{
{name}_buffer.emplace_back(*{name}_ptr ? CreateDateTime(**{name}_ptr) : nonstd::optional<{cpp_type}>());
++{name}_ptr;
}}
""", ).format(
name=arg_name,
items_var_name=items_var_name,
cpp_type=self.CppType,
)
buffer_type = "std::vector<nonstd::optional<{}>>".format(
self.CppType)
else:
param_decorator = ""
validation_suffix = textwrap.dedent(
"""\
std::vector<{cpp_type}> {name}_buffer;
{name}_buffer.reserve({items_var_name});
DateTimeParameter const * const {name}_end({name}_ptr + {items_var_name});
while({name}_ptr != {name}_end) {{
{name}_buffer.emplace_back(CreateDateTime(*{name}_ptr));
++{name}_ptr;
}}
""", ).format(
name=arg_name,
items_var_name=items_var_name,
cpp_type=self.CppType,
)
buffer_type = "std::vector<{}>".format(self.CppType)
parameters[0].Type = "DateTimeParameter const * {}".format(
param_decorator)
return self.Result(
parameters,
textwrap.dedent(
"""\
if({name}_ptr == nullptr) throw std::invalid_argument("'{name}_ptr' is null");
if({items_var_name} == 0) throw std::invalid_argument("'{items_var_name}' is 0");
{validation_suffix}
""", ).format(
name=arg_name,
items_var_name=items_var_name,
validation_suffix="\n{}\n".format(validation_suffix, )
if validation_suffix else "",
),
invocation_template.format(
"{name}_buffer.data(), {name}_buffer.size()".format(
name=arg_name, ), ),
input_buffer_type=self.Type(buffer_type,
"{}_buffer".format(arg_name)),
)
# ----------------------------------------------------------------------
@Interface.override
def GetOutputInfo(
self,
arg_name,
result_name="result",
suppress_pointer=False,
):
raise NotImplemented("Not implemented yet")
# ----------------------------------------------------------------------
@Interface.override
def GetDestroyOutputInfo(
self,
arg_name="result",
):
raise NotImplemented("Not implemented yet")
