def type_to_typelist(type_obj): if type_obj.IsArray: return PythonScraper.type_to_typelist(tuple) elif type_obj == Void: return PythonScraper.type_to_typelist(type(None)) elif not PythonOps.IsPythonType(clr.GetPythonType(type_obj)): raise NonPythonTypeException return PythonScraper.type_to_typelist(clr.GetPythonType(type_obj))
def get_ret_type(ret_type, obj_class, mod): if ret_type is not None: if ret_type == 'copy' and obj_class is not None: # returns a copy of self return PythonScraper.type_to_typelist(obj_class) else: return [type_name_to_typeref(ret_type, mod, RETURN_TYPE_OVERRIDES)]
def get_new_overloads(type_obj, obj): try: type_doc = safe_getattr(type_obj, '__doc__', None) type_type = type(type_obj) except: return None res = get_overloads_from_doc_string( type_doc, safe_getattr(type_obj, '__module__', None), type_type, safe_getattr(type_obj, '__name__', None), [{'type': PythonScraper.type_to_typelist(type), 'name': 'cls'}], ) if not res: obj_doc = safe_getattr(obj, '__doc__', None) if not obj_doc: return None res = get_overloads_from_doc_string( obj_doc, safe_getattr(type_obj, '__module__', None), type_type, safe_getattr(type_obj, '__name__', None), ) return res
def get_ret_type(ret_type, obj_class, mod): if ret_type is not None: if ret_type == 'copy' and obj_class is not None: # returns a copy of self return PythonScraper.type_to_name(obj_class) else: return type_name_to_type(ret_type, mod, RETURN_TYPE_OVERRIDES)
def get_new_overloads(type_obj, obj): try: type_doc = safe_getattr(type_obj, '__doc__', None) type_type = type(type_obj) except: return None res = get_overloads_from_doc_string( type_doc, safe_getattr(type_obj, '__module__', None), type_type, safe_getattr(type_obj, '__name__', None), [{ 'type': PythonScraper.type_to_typelist(type), 'name': 'cls' }], ) if not res: obj_doc = safe_getattr(obj, '__doc__', None) if not obj_doc: return None res = get_overloads_from_doc_string( obj_doc, safe_getattr(type_obj, '__module__', None), type_type, safe_getattr(type_obj, '__name__', None), ) return res
def get_overloads(func, is_method = False): if is_method: extra_args = [{'type': PythonScraper.type_to_name(object), 'name': 'self'}] else: extra_args = [] return get_overloads_from_doc_string(func.__doc__, getattr(func, '__module__', None), getattr(func, '__objclass__', None), getattr(func, '__name__', None), extra_args)
def type_name_to_typeref(name, mod, type_overrides = TYPE_OVERRIDES): arg_type = type_overrides.get(name, None) if arg_type is None: if name in BUILTIN_TYPES: arg_type = PythonScraper.type_to_typeref(get_builtin(name)) elif mod is not None and name in mod.__dict__: arg_type = PythonScraper.typename_to_typeref(mod.__name__, name) elif name.startswith('list'): arg_type = PythonScraper.type_to_typeref(list) else: # see if we can find it in any module we've imported... for mod_name, mod in sys.modules.items(): if mod is not None and name in mod.__dict__ and isinstance(mod.__dict__[name], type): arg_type = PythonScraper.typename_to_typeref(mod_name, name) break else: first_space = name.find(' ') if first_space != -1: return type_name_to_typeref(name[:first_space], mod, type_overrides) arg_type = PythonScraper.typename_to_typeref(name) return arg_type
def get_new_overloads(type_obj, obj): res = get_overloads_from_doc_string(type_obj.__doc__, getattr(type_obj, '__module__', None), type(type_obj), getattr(type_obj, '__name__', None), [{'type': PythonScraper.type_to_name(type), 'name': 'cls'}]) if not res: res = get_overloads_from_doc_string(obj.__doc__, getattr(type_obj, '__module__', None), type(type_obj), getattr(type_obj, '__name__', None)) return res
def type_name_to_typeref(name, mod, type_overrides=TYPE_OVERRIDES): arg_type = type_overrides.get(name, None) if arg_type is None: if name in BUILTIN_TYPES: arg_type = PythonScraper.type_to_typeref(get_builtin(name)) elif mod is not None and name in mod.__dict__: arg_type = PythonScraper.typename_to_typeref(mod.__name__, name) elif name.startswith('list'): arg_type = PythonScraper.type_to_typeref(list) else: # see if we can find it in any module we've imported... for mod_name, mod in list(sys.modules.items()): if mod is not None and name in mod.__dict__ and isinstance( mod.__dict__[name], type): arg_type = PythonScraper.typename_to_typeref( mod_name, name) break else: first_space = name.find(' ') if first_space != -1: return type_name_to_typeref(name[:first_space], mod, type_overrides) arg_type = PythonScraper.typename_to_typeref(name) return arg_type
def type_name_to_type(name, mod, type_overrides = TYPE_OVERRIDES): arg_type = type_overrides.get(name, None) if arg_type is None: if name in BUILTIN_TYPES: arg_type = PythonScraper.type_to_name(get_builtin(name)) elif mod is not None and name in mod.__dict__: arg_type = PythonScraper.memoize_type_name((mod.__name__, name)) elif name.startswith('list'): arg_type = PythonScraper.type_to_name(list) elif name == 'unicode': # Py3k, some doc strings still have unicode in them. arg_type = PythonScraper.type_to_name(str) else: # see if we can find it in any module we've imported... for mod_name, mod in sys.modules.items(): if mod is not None and name in mod.__dict__ and isinstance(mod.__dict__[name], type): arg_type = (mod_name, name) break else: first_space = name.find(' ') if first_space != -1: return type_name_to_type(name[:first_space], mod, type_overrides) arg_type = ('', name) return arg_type
def get_overloads(func, is_method = False): if is_method: extra_args = [{'type': PythonScraper.type_to_typelist(object), 'name': 'self'}] else: extra_args = [] func_doc = safe_getattr(func, '__doc__', None) if not func_doc: return None return get_overloads_from_doc_string( func_doc, safe_getattr(func, '__module__', None), safe_getattr(func, '__objclass__', None), safe_getattr(func, '__name__', None), extra_args, )
def get_overloads(func, is_method=False): if is_method: extra_args = [{ 'type': PythonScraper.type_to_typelist(object), 'name': 'self' }] else: extra_args = [] func_doc = safe_getattr(func, '__doc__', None) if not func_doc: return None return get_overloads_from_doc_string( func_doc, safe_getattr(func, '__module__', None), safe_getattr(func, '__objclass__', None), safe_getattr(func, '__name__', None), extra_args, )
safe_getattr = PythonScraper.safe_getattr BUILTIN_TYPES = [ type_name for type_name in builtins_keys() if type(get_builtin(type_name)) is type ] if sys.version_info[0] >= 3: BUILTIN = 'builtins' unicode = str else: BUILTIN = '__builtin__' TYPE_OVERRIDES = { 'string': PythonScraper.type_to_typeref(types.CodeType), 's': PythonScraper.type_to_typeref(str), 'integer': PythonScraper.type_to_typeref(int), 'boolean': PythonScraper.type_to_typeref(bool), 'number': PythonScraper.type_to_typeref(int), 'pid': PythonScraper.type_to_typeref(int), 'ppid': PythonScraper.type_to_typeref(int), 'fd': PythonScraper.type_to_typeref(int), 'handle': PythonScraper.type_to_typeref(int), 'Exit': PythonScraper.type_to_typeref(int), 'fd2': PythonScraper.type_to_typeref(int), 'Integral': PythonScraper.type_to_typeref(int), 'exit_status': PythonScraper.type_to_typeref(int), 'old_mask': PythonScraper.type_to_typeref(int), 'source': PythonScraper.type_to_typeref(str), 'newpos': PythonScraper.type_to_typeref(int),
if mod_path and mod_path != '-': import os.path if os.path.exists(mod_path): sys.path.insert(0, mod_path) remove_sys_path_0 = True __import__(mod_name) module = sys.modules[mod_name] finally: if remove_sys_path_0: del sys.path[0] if not module: print('__import__("' + mod_name + '")') PythonScraper.write_analysis( output_path, { "members": {}, "doc": "Could not import compiled module" }) elif mod_path and mod_path != '-': try: import os.path mod_name = os.path.split(mod_path)[1].partition('.')[0] try: import importlib module = importlib.import_module(mod_name) except ImportError: # Don't really care which import failed - we'll try imp pass if not module: import imp module = imp.load_dynamic(mod_name, mod_path)
if isinstance(__builtins__, dict): return __builtins__[name] return getattr(__builtins__, name) safe_getattr = PythonScraper.safe_getattr BUILTIN_TYPES = [type_name for type_name in builtins_keys() if type(get_builtin(type_name)) is type] if sys.version_info[0] >= 3: BUILTIN = 'builtins' unicode = str else: BUILTIN = '__builtin__' TYPE_OVERRIDES = { 'string': PythonScraper.type_to_typeref(types.CodeType), 's': PythonScraper.type_to_typeref(str), 'integer': PythonScraper.type_to_typeref(int), 'boolean': PythonScraper.type_to_typeref(bool), 'number': PythonScraper.type_to_typeref(int), 'pid': PythonScraper.type_to_typeref(int), 'ppid': PythonScraper.type_to_typeref(int), 'fd': PythonScraper.type_to_typeref(int), 'handle': PythonScraper.type_to_typeref(int), 'Exit': PythonScraper.type_to_typeref(int), 'fd2': PythonScraper.type_to_typeref(int), 'Integral': PythonScraper.type_to_typeref(int), 'exit_status':PythonScraper.type_to_typeref(int), 'old_mask': PythonScraper.type_to_typeref(int), 'source': PythonScraper.type_to_typeref(str), 'newpos': PythonScraper.type_to_typeref(int),
return __builtins__.keys() return dir(__builtins__) def get_builtin(name): if isinstance(__builtins__, dict): return __builtins__[name] return getattr(__builtins__, name) BUILTIN_TYPES = [type_name for type_name in builtins_keys() if type(get_builtin(type_name)) is type] if sys.version >= '3.': BUILTIN = 'builtins' else: BUILTIN = '__builtin__' TYPE_OVERRIDES = {'string': PythonScraper.type_to_name(types.CodeType), 's': PythonScraper.type_to_name(str), 'integer': PythonScraper.type_to_name(int), 'boolean': PythonScraper.type_to_name(bool), 'number': PythonScraper.type_to_name(int), 'pid': PythonScraper.type_to_name(int), 'ppid': PythonScraper.type_to_name(int), 'fd': PythonScraper.type_to_name(int), 'handle': PythonScraper.type_to_name(int), 'Exit': PythonScraper.type_to_name(int), 'fd2': PythonScraper.type_to_name(int), 'Integral': PythonScraper.type_to_name(int), 'exit_status':PythonScraper.type_to_name(int), 'old_mask': PythonScraper.type_to_name(int), 'source': PythonScraper.type_to_name(str), 'newpos': PythonScraper.type_to_name(int),
remove_sys_path_0 = False try: if mod_path and mod_path != '-': import os.path if os.path.exists(mod_path): sys.path.insert(0, mod_path) remove_sys_path_0 = True __import__(mod_name) module = sys.modules[mod_name] finally: if remove_sys_path_0: del sys.path[0] if not module: print('__import__("' + mod_name + '")') PythonScraper.write_analysis(output_path, {"members": {}, "doc": "Could not import compiled module"}) elif mod_path and mod_path != '-': try: import os.path mod_name = os.path.split(mod_path)[1].partition('.')[0] try: import importlib module = importlib.import_module(mod_name) except ImportError: # Don't really care which import failed - we'll try imp pass if not module: import imp module = imp.load_dynamic(mod_name, mod_path) finally: if not module:
# Usage: ExtensionScraper.py scrape [mod_name or '-'] [mod_path or '-'] [output_path] if len(sys.argv) != 5 or sys.argv[1].lower() != 'scrape': raise ValueError('Expects "ExtensionScraper.py scrape [mod_name|'-'] [mod_path|'-'] [output_path]"') mod_name, mod_path, output_path = sys.argv[2:] module = None if mod_name and mod_name != '-': try: __import__(mod_name) module = sys.modules[mod_name] finally: if not module: print('__import__("' + mod_name + '")') elif mod_path and mod_path != '-': try: import imp import os.path mod_name = os.path.splitext(os.path.split(mod_path)[1])[0] module = imp.load_dynamic(mod_name, mod_path) finally: if not module: print('imp.load_dynamic("' + mod_name + '", "' + mod_path + '")') else: raise ValueError('No module name or path provided') import PythonScraper analysis = PythonScraper.generate_module(module) PythonScraper.write_analysis(output_path, analysis)