def get_pch_include(file_path):
print 'file_path', file_path
ssk_path = os.path.join(git_repo_path, 'SignalServiceKit') + os.sep
sm_path = os.path.join(git_repo_path, 'SignalMessaging') + os.sep
s_path = os.path.join(git_repo_path, 'Signal') + os.sep
sae_path = os.path.join(git_repo_path, 'SignalShareExtension') + os.sep
print 'ssk_path', ssk_path
print 'sm_path', sm_path
print 's_path', s_path
print 'sae_path', sae_path
if file_path.startswith(ssk_path):
return os.path.join(
git_repo_path,
"Pods/Target Support Files/SignalServiceKit/SignalServiceKit-prefix.pch"
)
elif file_path.startswith(sm_path):
return os.path.join(git_repo_path,
"SignalMessaging/SignalMessaging-Prefix.pch")
elif file_path.startswith(s_path):
return os.path.join(git_repo_path, "Signal/Signal-Prefix.pch")
elif file_path.startswith(sae_path):
return os.path.join(
git_repo_path,
"SignalShareExtension/SignalShareExtension-Prefix.pch")
else:
fail("Couldn't determine .pch for file:", file_path)
def process_objc_property_impl(clazz, prefix, file_path, line, remainder):
# print '\t\t', 'process_objc_property_impl', remainder
match = process_objc_property_impl_regex.search(remainder)
if match is None:
print 'file_path:', file_path
fail('Could not match line:', line)
property_name = match.group(1).strip()
property = clazz.get_property(property_name)
if property is None:
if clazz.name == 'AppDelegate' and property_name == 'window':
# We can't parse properties synthesized locally but
# declared in a protocol defined in the iOS frameworks.
# So, special case these propert(y/ies) - we don't need
# to handle them.
return
print 'file_path:', file_path
print 'line:', line
print '\t', 'clazz', clazz.name, clazz.counter
print '\t', 'property_name', property_name
for name in clazz.property_names():
print '\t\t', name
fail("Can't find property:", property_name)
else:
print 'property synthesized', line
property.is_synthesized = True
def process_objc_type_declaration(namespace, file_path, lines, prefix, remainder):
print '\t', 'type_declaration', remainder
match = process_objc_type_declaration_regex.search(remainder)
if match is None:
print 'file_path:', file_path
fail('Could not match line:', remainder)
type1 = get_match_group(match, 1)
type2 = get_match_group(match, 2)
type3 = get_match_group(match, 3)
if type1 is None or type3 is None:
return
is_enum = (type1 == 'enum ' + type3)
if not is_enum:
return
# print 'type_declaration:', type1, type2, type3
if type3.startswith('line:'):
print 'Ignoring invalid enum(2):', type1, type2, type3
return
if type3 not in enum_type_map:
print 'Enum has unknown type:', type3
enum_type = 'NSUInteger'
else:
enum_type = enum_type_map[type3]
enum_type_map[type3] = enum_type
def process_objc_category(namespace, file_path, lines, decl_prefix,
decl_remainder):
# print '\t', 'category'
# |-ObjCCategoryDecl 0x1092f8440 <line:76:1, line:81:2> line:76:12 SomeCategory
# | |-ObjCInterface 0x1092f5d58 'ObjCMessageWAuthor'
# | |-ObjCCategoryImpl 0x1092f8608 'SomeCategory'
# | |-ObjCPropertyDecl 0x1092f8508 <line:79:1, col:53> col:53 fakeProperty2 'NSString * _Nullable':'NSString *' readonly nonatomic
# | `-ObjCMethodDecl 0x1092f8580 <col:53> col:53 implicit - fakeProperty2 'NSString * _Nullable':'NSString *'
if not lines.hasNext():
fail('Category missing interface.')
line = lines.next()
prefix, remainder = split_objc_ast_prefix(line)
if len(prefix) <= len(decl_prefix):
fail('Category missing interface.')
class_name = remainder.split(' ')[-1]
if class_name.startswith("'") and class_name.endswith("'"):
class_name = class_name[1:-1]
process_objc_class(namespace,
file_path,
lines,
decl_prefix,
decl_remainder,
custom_class_name=class_name)
def class_protocols(self):
result = []
for protocol_name in self.protocol_names:
if protocol_name == self.name:
# There are classes that implement a protocol of the same name, e.g. MTLModel
# Ignore them.
continue
# print 'get_inherited_property:', name, self.name, protocol_name
protocol = self.namespace.find_class(protocol_name)
if protocol is None:
if protocol_name.startswith('NS') or protocol_name.startswith(
'AV') or protocol_name.startswith(
'UI') or protocol_name.startswith(
'MF') or protocol_name.startswith(
'UN') or protocol_name.startswith('CN'):
# Ignore built in protocols.
continue
print 'clazz:', self.name
print 'file_path:', file_path
fail('Missing protocol:', protocol_name)
result.append(protocol)
return result
def split_objc_ast_prefix(line):
match = split_objc_ast_prefix_regex.search(line)
if match is None:
fail('Could not match line:', line)
prefix = match.group(1)
remainder = match.group(2)
# print 'prefix', prefix
# print 'remainder', remainder
return prefix, remainder
def process_objc_class(namespace, file_path, lines, decl_prefix, decl_remainder, custom_class_name=None, super_class_name=None):
if custom_class_name is not None:
class_name = custom_class_name
else:
class_name = decl_remainder.split(' ')[-1]
# print '\t', 'class_name', class_name
clazz = namespace.upsert_class(class_name)
if super_class_name is not None:
if clazz.super_class_name is None:
clazz.super_class_name = super_class_name
elif clazz.super_class_name != super_class_name:
fail("super_class_name does not match:", clazz.super_class_name, super_class_name)
while lines.hasNext():
line = lines.next()
prefix, remainder = split_objc_ast_prefix(line)
if len(prefix) <= len(decl_prefix):
# Declaration is over.
return clazz
line = lines.popNext()
# print '\t', 'interface', 'line', line
# print '\t', '\t', 'remainder', remainder
# | |-ObjCPropertyDecl 0x10f5d2d40 <line:19:1, col:43> col:43 uniqueId 'NSString * _Nonnull':'NSString *' readonly nonatomic
"""
TODO: We face interesting choices about how to process:
* properties
* private properties
* properties with renamed ivars
* ivars without properties
* properties not backed by ivars (e.g. actually accessors).
"""
# print 'remainder', remainder
if remainder.startswith('ObjCPropertyDecl '):
# print 'property ?', line
process_objc_property(clazz, prefix, file_path, line, remainder)
elif remainder.startswith('ObjCPropertyImplDecl '):
# print 'property impl', line
process_objc_property_impl(clazz, prefix, file_path, line, remainder)
elif remainder.startswith('ObjCMethodDecl '):
# print 'property impl', line
process_objc_method_decl(clazz, prefix, file_path, line, remainder)
elif remainder.startswith('ObjCProtocol '):
# print 'property impl', line
process_objc_protocol(namespace, clazz, prefix, file_path, line, remainder)
return clazz
def process_objc_ast(namespace: Namespace, file_path: str,
raw_ast: str) -> None:
m_filename = os.path.basename(file_path)
print('m_filename:', m_filename)
file_base, file_extension = os.path.splitext(m_filename)
if file_extension != '.m':
fail('Bad file extension:', file_extension)
h_filename = file_base + '.h'
print('h_filename:', h_filename)
# TODO: Remove
lines = raw_ast.split('\n')
# lines = lines[-1000:]
raw_ast = '\n'.join(lines)
lines = LineProcessor(raw_ast)
while lines.hasNext():
line = lines.popNext()
# print 'line:', line
prefix, remainder = split_objc_ast_prefix(line)
if remainder.startswith('ObjCInterfaceDecl '):
# |-ObjCInterfaceDecl 0x112510490 <SignalDataStoreCommon/ObjCMessage.h:14:1, line:25:2> line:14:12 ObjCMessage
# print 'interface', line
process_objc_interface(namespace, file_path, lines, prefix,
remainder)
elif remainder.startswith('ObjCCategoryDecl '):
# |-ObjCCategoryDecl 0x112510d58 <SignalDataStoreCommon/ObjCMessage.m:18:1, line:22:2> line:18:12
# print 'category', line
process_objc_category(namespace, file_path, lines, prefix,
remainder)
elif remainder.startswith('ObjCImplementationDecl '):
# `-ObjCImplementationDecl 0x112510f20 <line:24:1, line:87:1> line:24:17 ObjCMessage
# print 'implementation', line
process_objc_implementation(namespace, file_path, lines, prefix,
remainder)
elif remainder.startswith('ObjCProtocolDecl '):
# `-ObjCImplementationDecl 0x112510f20 <line:24:1, line:87:1> line:24:17 ObjCMessage
# print 'implementation', line
process_objc_protocol_decl(namespace, file_path, lines, prefix,
remainder)
# TODO: Category impl.
elif remainder.startswith('TypedefDecl '):
# `-ObjCImplementationDecl 0x112510f20 <line:24:1, line:87:1> line:24:17 ObjCMessage
# print 'implementation', line
process_objc_type_declaration(namespace, file_path, lines, prefix,
remainder)
elif remainder.startswith('EnumDecl '):
# `-ObjCImplementationDecl 0x112510f20 <line:24:1, line:87:1> line:24:17 ObjCMessage
# print 'implementation', line
process_objc_enum_declaration(namespace, file_path, lines, prefix,
remainder)
def process_file(file_path, namespace, intermediates):
filename = os.path.basename(file_path)
if not filename.endswith('.swift'):
return
if filename == 'EmojiWithSkinTones+String.swift':
return
command = [
'which',
'sourcekitten',
]
exit_code, output, error_output = ows_getoutput(command)
if exit_code != 0:
fail('Missing sourcekitten. Install with homebrew?')
print 'Extracting Swift Bridging Info For:', file_path
command = [
'sourcekitten',
'structure',
'--file',
file_path,
]
# for part in command:
# print '\t', part
# command = ' '.join(command).strip()
# print 'command', command
# output = commands.getoutput(command)
# command = ' '.join(command).strip()
# print 'command', command
exit_code, output, error_output = ows_getoutput(command)
if exit_code != 0:
print 'exit_code:', exit_code
if len(error_output.strip()) > 0:
print 'error_output:', error_output
# print 'output:', len(output)
# exit(1)
output = output.strip()
# print 'output', output
if intermediates:
intermediate_file_path = file_path + '.ast'
print 'Writing intermediate:', intermediate_file_path
with open(intermediate_file_path, 'wt') as f:
f.write(output)
parse_swift_ast(file_path, namespace, output)
def parse_swift_ast(file_path, namespace, ast):
json_data = json.loads(ast)
json_maps = json_data.get('key.substructure')
if json_maps is None:
return
for json_map in json_maps:
kind = json_map.get('key.kind')
if kind is None:
continue
elif kind == 'source.lang.swift.decl.protocol':
# "key.kind" : "source.lang.swift.decl.protocol",
# "key.length" : 1067,
# "key.name" : "TypingIndicators",
# "key.namelength" : 16,
# "key.nameoffset" : 135,
# "key.offset" : 126,
# "key.runtime_name" : "OWSTypingIndicators",
name = json_map.get('key.runtime_name')
if name is None or len(name) < 1 or name.startswith('_'):
name = json_map.get('key.name')
if name is None or len(name) < 1:
fail('protocol is missing name.')
continue
if name.startswith('_'):
continue
namespace.swift_protocol_names.append(name)
elif kind == 'source.lang.swift.decl.class':
# "key.kind" : "source.lang.swift.decl.class",
# "key.length" : 15057,
# "key.name" : "TypingIndicatorsImpl",
# "key.namelength" : 20,
# "key.nameoffset" : 1251,
# "key.offset" : 1245,
# "key.runtime_name" : "OWSTypingIndicatorsImpl",
name = json_map.get('key.runtime_name')
if name is None or len(name) < 1 or name.startswith('_'):
name = json_map.get('key.name')
if name is None or len(name) < 1:
fail('class is missing name.')
continue
if name.startswith('_'):
continue
namespace.swift_class_names.append(name)
def process_objc(file_path, swift_bridging_path, intermediates):
module_header_dir_path = gather_module_headers('Pods')
pch_include = get_pch_include(file_path)
# These clang args can be found by building our workspace and looking at how XCode invokes clang.
clang_args = '-arch arm64 -fmessage-length=0 -fdiagnostics-show-note-include-stack -fmacro-backtrace-limit=0 -std=gnu11 -fobjc-arc -fobjc-weak -fmodules -gmodules -fmodules-prune-interval=86400 -fmodules-prune-after=345600 -Wnon-modular-include-in-framework-module -Werror=non-modular-include-in-framework-module -fapplication-extension -Wno-trigraphs -fpascal-strings -O0 -fno-common -Wno-missing-field-initializers -Wno-missing-prototypes -Werror=return-type -Wdocumentation -Wunreachable-code -Wno-implicit-atomic-properties -Werror=deprecated-objc-isa-usage -Wno-objc-interface-ivars -Werror=objc-root-class -Wno-arc-repeated-use-of-weak -Wimplicit-retain-self -Wduplicate-method-match -Wno-missing-braces -Wparentheses -Wswitch -Wunused-function -Wno-unused-label -Wno-unused-parameter -Wunused-variable -Wunused-value -Wempty-body -Wuninitialized -Wconditional-uninitialized -Wno-unknown-pragmas -Wno-shadow -Wno-four-char-constants -Wno-conversion -Wconstant-conversion -Wint-conversion -Wbool-conversion -Wenum-conversion -Wno-float-conversion -Wnon-literal-null-conversion -Wobjc-literal-conversion -Wshorten-64-to-32 -Wpointer-sign -Wno-newline-eof -Wno-selector -Wno-strict-selector-match -Wundeclared-selector -Wdeprecated-implementations'.split(' ')
command = [
'xcrun',
'--show-sdk-path',
'--sdk',
'iphoneos',
]
exit_code, output, error_output = ows_getoutput(command)
if int(exit_code) != 0:
fail('Could not find iOS SDK.')
iphoneos_sdk_path = output.strip()
# TODO: We'll never repro the correct search paths, so clang will always emit errors.
# We'll want to ignore these errors without silently failing.
command = [
'clang',
'-x',
'objective-c',
'-Xclang',
'-ast-dump',
'-fobjc-arc',
] + clang_args + [
'-isysroot',
iphoneos_sdk_path,
] + find_header_include_paths('SignalServiceKit/src') + [
] + find_header_include_paths('SignalMessaging') + [
# ] + find_header_include_paths('Pods') + [
('-I' + module_header_dir_path),
('-I' + swift_bridging_path),
'-I/Users/matthew/code/workspace/ows/Signal-iOS-2/Pods/SignalCoreKit/SignalCoreKit',
'-include',
pch_include,
file_path,
# '|',
# 'sed',
# "$'s,\x1b\\[[0-9;]*[a-zA-Z],,g'",
# '>',
# 'test4.txt'
]
for part in command:
print '\t', part
# command = ' '.join(command).strip()
# print 'command', command
# output = commands.getoutput(command)
# command = ' '.join(command).strip()
print 'command', command
exit_code, output, error_output = ows_getoutput(command)
print 'exit_code:', exit_code
print 'error_output:', error_output
# print 'output:', len(output)
# exit(1)
output = output.strip()
raw_ast = output
if intermediates:
intermediate_file_path = file_path + '.ast'
print 'Writing intermediate:', intermediate_file_path
with open(intermediate_file_path, 'wt') as f:
f.write(raw_ast)
print 'raw_ast:', len(raw_ast)
namespace = Namespace()
process_objc_ast(namespace, file_path, raw_ast)
output = emit_output(file_path, namespace)
print 'output', output
parsed_file_path = file_path + sds_common.SDS_JSON_FILE_EXTENSION
print 'parsed_file_path', parsed_file_path
with open(parsed_file_path, 'wt') as f:
f.write(output)
def process_objc_property(clazz, prefix, file_path, line, remainder):
match = process_objc_property_regex.search(remainder)
if match is None:
print 'file_path:', file_path
print 'remainder:', remainder
fail('Could not match line:', line)
property_name = match.group(1).strip()
property_type_1 = get_match_group(match, 2)
property_type_2 = get_match_group(match, 4)
property_keywords = match.group(5).strip().split(' ')
is_optional = (property_type_2 + ' _Nullable') == property_type_1
is_readonly = 'readonly' in property_keywords
property_type = property_type_2
if len(property_type_2) < 1:
property_type = property_type_1
primitive_types = (
'BOOL',
'NSInteger',
'NSUInteger',
'uint64_t',
'int64_t'
)
if property_type_1 in primitive_types:
property_type = property_type_1
# property_type = property_type_1
print '\t', property_name, 'property_type', property_type, 'property_type1', property_type_1, 'property_type2', property_type_2, 'property_type', property_type
property = clazz.get_property(property_name)
if property is None:
property = ParsedProperty(property_name, property_type, is_optional)
# print 'property found', line
clazz.add_property(property)
else:
if property.name != property_name:
fail("Property names don't match", property.name, property_name)
if property.is_optional != is_optional:
if clazz.name.startswith('DD'):
# CocoaLumberjack has nullability consistency issues.
# Ignore them.
return
print 'file_path:', file_path
print 'clazz:', clazz.name
fail("Property is_optional don't match", property_name)
if property.objc_type != property_type:
# There's a common pattern of using a mutable private property
# and exposing a non-mutable public property to prevent
# external modification of the property.
if property_type.startswith('NSMutable') and property.objc_type == 'NS' + property_type[len('NSMutable'):]:
property.objc_type = property_type
else:
print 'file_path:', file_path
print 'remainder:', remainder
print 'property.objc_type:', property.objc_type
print 'property_type:', property_type
print 'property_name:', property_name
fail("Property types don't match", property.objc_type, property_type)
if not is_readonly:
property.is_not_readonly = True
