def Deserialize_FileInfo(
items,
process_additional_data=False,
always_include_optional=False,
is_root=False,
):
"""Deserializes 'FileInfo' from a JSON object to a python object"""
if isinstance(items, six.string_types):
if FileSystem.IsFilename(items):
with open(items) as f:
items = json.load(f)
else:
items = json.loads(items)
if not isinstance(items, list):
if isinstance(items, dict) and "FileInfo" in items:
items = items["FileInfo"]
elif not isinstance(items, dict) and hasattr(items, "FileInfo"):
items = getattr(items, "FileInfo")
elif is_root:
items = DoesNotExist
try:
try:
items = Deserializer().FileInfo(
items,
process_additional_data=process_additional_data,
always_include_optional=always_include_optional,
)
if items is DoesNotExist:
items = []
except:
_DecorateActiveException("FileInfo")
except SerializationException:
raise
except Exception as ex:
raise DeserializeException(ex)
return items
def Deserialize(
root,
process_additional_data=False,
always_include_optional=False,
):
"""Convenience method that deserializes all top-level elements"""
if isinstance(root, six.string_types):
if FileSystem.IsFilename(root):
with open(root) as f:
root = rtyaml.load(f)
else:
root = rtyaml.load(root)
result = _CreatePythonObject(attributes=None, )
this_result = Deserialize_filter(
root,
is_root=True,
process_additional_data=process_additional_data,
always_include_optional=always_include_optional,
)
if this_result is not DoesNotExist:
setattr(result, "filter", this_result)
elif always_include_optional:
setattr(result, "filter", None)
this_result = Deserialize_named_filters(
root,
is_root=True,
process_additional_data=process_additional_data,
always_include_optional=always_include_optional,
)
if this_result is not DoesNotExist:
setattr(result, "named_filters", this_result)
elif always_include_optional:
setattr(result, "named_filters", [])
return result
def Deserialize_filter(
item,
process_additional_data=False,
always_include_optional=False,
is_root=False,
):
"""Deserializes 'filter' from a YAML object to a python object"""
if isinstance(item, six.string_types):
if FileSystem.IsFilename(item):
with open(item) as f:
item = rtyaml.load(f)
else:
item = rtyaml.load(item)
if not isinstance(item, list):
if isinstance(item, dict) and "filter" in item:
item = item["filter"]
elif not isinstance(item, dict) and hasattr(item, "filter"):
item = getattr(item, "filter")
elif is_root:
item = DoesNotExist
try:
try:
item = Deserializer().filter(
item,
process_additional_data=process_additional_data,
always_include_optional=always_include_optional,
)
except:
_DecorateActiveException("filter")
except SerializationException:
raise
except Exception as ex:
raise DeserializeException(ex)
return item
def Format(cls,
filename_or_content,
include_plugin_names=None,
exclude_plugin_names=None,
debug=False,
hint_filename=None,
*plugin_input_dirs,
**plugin_args):
cls.__clsinit__(*plugin_input_dirs)
if FileSystem.IsFilename(filename_or_content):
with open(filename_or_content) as f:
filename_or_content = f.read()
input_content = filename_or_content
del filename_or_content
include_plugin_names = set(include_plugin_names or [])
exclude_plugin_names = set(exclude_plugin_names or [])
if debug:
if include_plugin_names:
include_plugin_names.add(cls._debug_plugin.Name)
else:
exclude_plugin_names.add(cls._debug_plugin.Name)
plugins = [
plugin for plugin in cls._plugins
if plugin.Name not in exclude_plugin_names and
(not include_plugin_names or plugin.Name in include_plugin_names)
]
# Preprocess the content
lines = input_content.split("\n")
for plugin in plugins:
lines = plugin.PreprocessLines(lines)
input_content = "\n".join(lines)
# Invoke clang-format
command_line = "clang-format -style=file{}".format(
' "-assume-filename={}"'.format(hint_filename)
if hint_filename is not None else "", )
result, output = Process.Execute(
command_line,
stdin=input_content,
)
if result != 0:
raise Exception(
textwrap.dedent(
"""\
clang-format failed: {}
{}
""", ).format(result, StringHelpers.LeftJustify(output,
4)), )
# Convert the lines into line structures
lines = []
continuation_block_index = 0
continuation_block_id = None
for line_content in output.split("\n"):
has_continuation_token = False
if line_content.endswith("\\"):
if continuation_block_id is None:
continuation_block_id = continuation_block_index
continuation_block_index += 1
has_continuation_token = True
line_content = line_content[:-1].rstrip()
line = PluginBase.Line(
line_content,
continuation_block_id=continuation_block_id,
)
if not has_continuation_token:
continuation_block_id = None
lines.append(line)
# Decorate the lines
for plugin in plugins:
args = []
kwargs = {}
defaults = plugin_args.get(plugin.Name, None)
if defaults is not None:
if isinstance(defaults, (list, tuple)):
args = defaults
elif isinstance(defaults, dict):
kwargs = defaults
else:
assert False, defaults
lines = plugin.Decorate(lines, *args, **kwargs)
# Postprocess the lines
for plugin in plugins:
lines = plugin.PostprocessLines(lines)
# Restore the line continuation chars
output = []
continuation_block_id = None
for line_index, line in enumerate(lines):
# Add continuation chars
if line.continuation_block_id is not None and line.continuation_block_id != continuation_block_id:
continuation_block_id = line.continuation_block_id
# Process all the lines in this block
max_line_length = 0
end_block_index = line_index
while end_block_index < len(lines) and lines[
end_block_index].continuation_block_id == continuation_block_id:
max_line_length = max(max_line_length,
len(lines[end_block_index].content))
end_block_index += 1
max_line_length += 2
index = line_index
while index < end_block_index:
if index + 1 == end_block_index:
suffix = ""
else:
suffix = "{}\\".format(
" " *
(max_line_length - len(lines[index].content)))
lines[index].content = "{}{}".format(
lines[index].content, suffix)
index += 1
output = "\n".join([line.content for line in lines])
return output, output != input_content
def ObtainFunctions(
input_filename,
on_unsupported,
policy_func,
):
"""
This function will extract return value, name and parameters for every
function given. input_filename can be a file name or a string that is the code
itself.
Return value:
Returns a list of functions, every item in this list is a dictionary that
has information about the function.
"""
is_temp_file = False
# Since clang can only parse from a file, if we are given a string we need to create
# a new temp file and put the string inside.
if not fileSystem.IsFilename(input_filename):
is_temp_file = True
file_content = input_filename
input_filename = CE_Shell.Shell.CreateTempFilename(suffix=".cpp")
with open(input_filename, "w") as file_pointer:
file_pointer.write(file_content)
# ----------------------------------------------------------------------
def DeleteFile():
if is_temp_file:
os.remove(input_filename)
# ----------------------------------------------------------------------
def TestAndVerify(types, verify_struct_func):
return policy_func(types, verify_struct_func)
# ----------------------------------------------------------------------
with callOnExit.CallOnExit(DeleteFile):
index = cindex.Index.create()
args = []
# On Windows, clang recognizes the INCLUDE and LIB environment variables, but
# on Linux it does not. Recognize the INCLUDE var on Linux.
if CurrentShell.CategoryName == "Linux":
include_vars = [
value for value in os.getenv("INCLUDE").split(":")
if value.strip()
]
# In the past, we used the GCC repository which would prepopulate the INCLUDE environment
# variable with standard includes. However, that repo had issues when attempting to link
# compile binaries and has been temporarily disabled. Unfortunately, this code needs that
# populated environment variable to function correctly in order to tokenize input.
#
# For now, add those includes here. When Clang's dependency on GCC is restored in the future,
# we can remove this explicit step.
include_vars.append("/usr/lib/gcc/x86_64-linux-gnu/7/include")
args = ['-I{}'.format(include_var) for include_var in include_vars]
pattern_const = re.compile(
textwrap.dedent(r"""(?#
Not a letter)(?<!\w)(?#
Keyword)(?P<keyword>const)(?#
Not a letter)(?!\w)(?#
)"""))
pattern_amper = re.compile("&*(& )*")
# ----------------------------------------------------------------------
def SimpleVarType(name):
"""
Remove 'const' and '&'
"""
name = re.sub(pattern_const, "", name)
name = re.sub(pattern_amper, "", name)
return name.strip()
# ----------------------------------------------------------------------
def ParseFile(filename):
"""
Clang opens all files that are included in 'filename' at the same time. Only the functions on
'filename' are processed, but all structs are processed, because one of the functions
in 'filename' might need to use it. The ones that are not in 'filename' and are not used are not
exported.
"""
translation_unit = index.parse(filename,
args=args + ['-std=c++17'])
diagnostics = list(translation_unit.diagnostics)
if diagnostics:
raise Exception("\n".join([str(diag) for diag in diagnostics]))
cursor = translation_unit.cursor
# ----------------------------------------------------------------------
def GetAlias(node, prefix):
"""
This function will process all 'typedef' and 'using' and it will map the underlying type to
its definition.
"""
alias = {}
if node.kind == cindex.CursorKind.NAMESPACE:
for child in node.get_children():
ret = GetAlias(child, prefix + node.spelling + "::")
for a, b in ret.items():
assert prefix + a not in alias.keys()
alias[prefix + a] = b
if (node.kind == cindex.CursorKind.TYPEDEF_DECL
or node.kind == cindex.CursorKind.TYPE_ALIAS_DECL):
alias[
prefix +
node.spelling] = node.underlying_typedef_type.spelling
return alias
# ----------------------------------------------------------------------
alias = {}
for node in cursor.get_children():
for before_type, after_type in GetAlias(node, "").items():
assert before_type not in alias.keys()
'''
If TestAndVerify with no structs (lambda function is always False) is True, then it means
that before_type is already an accepted type, so we don't want to extend it any further.
'''
if not TestAndVerify(before_type, lambda type: False):
alias[before_type] = after_type
alias_regex = re.compile(
textwrap.dedent(r"""(?#
Not a letter or a ':')(?<![:\w])(?#
Keyword)(?P<keyword>{})(?#
Not a letter or a ':')(?![:\w])(?#
)""").format("|".join([re.escape(key) for key in alias])))
struct_pattern = re.compile(
textwrap.dedent(r"""(?#
Not a letter)(?<!\w)(?#
Keyword with a space)(?P<keyword>struct\s)(?#
)"""))
# ----------------------------------------------------------------------
def FullVarType(types):
"""
This will undo all 'typedef' and 'using' by looking for the items in the 'alias' dict and substituting
the corresponding definitions. It will also remove all occurences of the words 'struct'.
"""
num_subs = True
while num_subs and alias:
types, num_subs = re.subn(alias_regex,
lambda k: alias[k.group(1)],
types)
types = struct_pattern.sub(r'', types)
return types
# ----------------------------------------------------------------------
struct_list = []
function_dict = {}
# ----------------------------------------------------------------------
def EnumerateStruct(node):
if node.kind == cindex.CursorKind.NAMESPACE:
for child in node.get_children():
EnumerateStruct(child)
if node.kind == cindex.CursorKind.STRUCT_DECL:
this_struct = _GetStruct(node, SimpleVarType, FullVarType)
if this_struct:
struct_list.append(this_struct)
# ----------------------------------------------------------------------
def EnumerateFuncs(node):
if node.kind == cindex.CursorKind.NAMESPACE:
for child in node.get_children():
EnumerateFuncs(child)
if node.kind == cindex.CursorKind.FUNCTION_DECL and node.location.file.name == filename:
ret_type = FullVarType(node.result_type.spelling)
arg_list = []
for arg in node.get_arguments():
arg_type = FullVarType(arg.type.spelling)
arg_list.append(arg_type)
# There is the need for arg_list to be a tuple here, because this will be the key
# to the dictionary of functions, and tuple are hashable unlike lists.
func_key = (_FullName(node), ret_type, tuple(arg_list))
is_def = False
for child in node.get_children():
if child.kind == cindex.CursorKind.COMPOUND_STMT:
is_def = True
if func_key not in function_dict.keys():
variable_info = []
for arg in node.get_arguments():
arg_type = FullVarType(arg.type.spelling)
variable_info.append((arg.displayname, arg_type,
SimpleVarType(arg_type)))
# Create the instance of the function
function_dict[func_key] = Function(
_FullName(node), ret_type, SimpleVarType(ret_type),
variable_info, node.location.line)
if is_def:
function_dict[
func_key].definition_line = node.location.line
# ----------------------------------------------------------------------
# EnumerateStruct needs to be separated from EnumerateFuncs because of constructors that might be out
# of the function.
for child in cursor.get_children():
EnumerateStruct(child)
for child in cursor.get_children():
EnumerateFuncs(child)
# ----------------------------------------------------------------------
def GetIncludeList(filename):
include_list = []
for child in translation_unit.get_includes():
include_list.append(
(os.path.realpath(str(child.location.file.name)) if
(not is_temp_file
or child.location.file.name != input_filename) else
None,
os.path.realpath(
os.path.join(filename, str(child.include)))))
return include_list
# ----------------------------------------------------------------------
include_list = GetIncludeList(filename)
function_list = [func for func in function_dict.values()]
return {
"function_list": function_list,
"struct_list": struct_list,
"include_list": include_list
}
clean_results = OrderedDict()
# ----------------------------------------------------------------------
def InitializeCleanResults(filename, raw_includes):
clean_results[filename] = Results()
for include_tuple in raw_includes:
name, include = include_tuple
if name == filename:
clean_results[filename].include_list.append(include)
# ----------------------------------------------------------------------
filename = os.path.realpath(input_filename)
these_results = ParseFile(filename)
# If the original file was a temp file, make the key None rather than
# the name of the temporary file used.
if not clean_results and is_temp_file:
filename = None
if not clean_results:
InitializeCleanResults(filename, these_results["include_list"])
needed_struct_list = []
invalid_struct_list = []
# ----------------------------------------------------------------------
def GetStruct(this_struct_name):
"""
Get Struct from its name.
"""
for this_struct in these_results["struct_list"]:
if this_struct.Name == this_struct_name:
return this_struct
return None
# ----------------------------------------------------------------------
def VerifyStruct(struct_name):
"""
Check all var types in this Struct, this Struct is valid if they are all valid. There is an assumption that
this function will only be called for an Struct that is required. If this Struct depends on another Struct,
that means that the other Struct is also required.
"""
this_struct = GetStruct(struct_name)
if this_struct is None or this_struct in invalid_struct_list:
return False
if this_struct in needed_struct_list:
return True
invalid_reasons = []
for var_type, var_name in zip(
this_struct.EnumerateSimpleVarTypes(),
this_struct.EnumerateVarNames()):
if GetStruct(var_type) != this_struct and not TestAndVerify(
var_type, VerifyStruct):
invalid_reasons.append(
"\t- Invalid var {} of type {}.".format(
var_name, var_type))
for constructor in this_struct.constructor_list:
for arg_type in constructor.EnumerateSimpleVarTypes():
if GetStruct(
arg_type) != this_struct and not TestAndVerify(
arg_type, VerifyStruct):
invalid_reasons.append(
"\t- Invalid type {} on constructor argument.".
format(arg_type))
for parent_struct in this_struct.base_structs:
if not VerifyStruct(parent_struct):
invalid_reasons.append(
"\t- Invalid base struct {}.".format(parent_struct))
if not this_struct.has_move_constructor:
invalid_reasons.append(
"\t- Struct doesn't have a move constructor.")
if this_struct.has_copy_constructor:
invalid_reasons.append("\t- Struct has a copy constructor.")
if this_struct.has_private:
invalid_reasons.append(
"\t- Struct has a private variable or inherits from a private struct."
)
if this_struct.has_other:
invalid_reasons.append(
"\t- Struct has an unsupported definition.")
if invalid_reasons:
on_unsupported(
textwrap.dedent("""\
The struct {} is not supported:
{}
""").format(this_struct.Name, "\n".join(invalid_reasons)),
this_struct.Filename if
(not is_temp_file
or this_struct.Filename != input_filename) else None,
this_struct.DefinitionLine)
invalid_struct_list.append(this_struct)
return False
needed_struct_list.append(this_struct)
return True
# ----------------------------------------------------------------------
def VerifyFunction(func, filename):
"""
A function is valid if all var types are valid.
"""
invalid_reasons = []
for var_type, var_name in zip(func.EnumerateSimpleVarTypes(),
func.EnumerateVarNames()):
if not TestAndVerify(var_type, lambda struct: False):
invalid_reasons.append(
"\t- Invalid argument {} of type {}.".format(
var_name, var_type))
return_type = func.SimpleReturnType
if not TestAndVerify(return_type, VerifyStruct):
invalid_reasons.append(
"\t- Invalid return type {}.".format(return_type))
if invalid_reasons:
on_unsupported(
textwrap.dedent("""\
The function {} is not supported:
{}
""").format(func.Name,
"\n".join(invalid_reasons)), filename if
(not is_temp_file or filename != input_filename) else None,
func.definition_line)
return False
return True
# ----------------------------------------------------------------------
for func in these_results["function_list"]:
if VerifyFunction(func, filename):
clean_results[filename].function_list.append(func.ToDict())
# Add required Struct to the clean_results list.
for this_struct in needed_struct_list:
this_file_name = this_struct.Filename
if is_temp_file:
this_file_name = None
if this_file_name not in clean_results:
InitializeCleanResults(this_file_name,
these_results["include_list"])
clean_results[this_file_name].struct_list.append(
this_struct.ToDict())
return dict([(filename, result.ToDict())
for filename, result in clean_results.items()])