def generate_literals(self, emitter: Emitter) -> None: emitter.emit_lines('static int CPyLiteralsInit(void)', '{', 'static int is_initialized = 0;', 'if (is_initialized) return 0;', '') for (_, literal), identifier in self.literals.items(): symbol = emitter.static_name(identifier, None) if isinstance(literal, int): emitter.emit_line( '{} = PyLong_FromString(\"{}\", NULL, 10);'.format( symbol, str(literal))) elif isinstance(literal, float): emitter.emit_line('{} = PyFloat_FromDouble({});'.format( symbol, str(literal))) elif isinstance(literal, str): emitter.emit_line( '{} = PyUnicode_FromStringAndSize({}, {});'.format( symbol, *encode_as_c_string(literal))) elif isinstance(literal, bytes): emitter.emit_line( '{} = PyBytes_FromStringAndSize({}, {});'.format( symbol, *encode_bytes_as_c_string(literal))) else: assert False, ( 'Literals must be integers, floating point numbers, or strings,', 'but the provided literal is of type {}'.format( type(literal))) emitter.emit_lines('if ({} == NULL)'.format(symbol), ' return -1;') emitter.emit_lines( 'is_initialized = 1;', 'return 0;', '}', )
def generate_wrapper_function(fn: FuncIR, emitter: Emitter, source_path: str, module_name: str) -> None: """Generates a CPython-compatible wrapper function for a native function. In particular, this handles unboxing the arguments, calling the native function, and then boxing the return value. """ emitter.emit_line('{} {{'.format(wrapper_function_header(fn, emitter.names))) # If we hit an error while processing arguments, then we emit a # traceback frame to make it possible to debug where it happened. # Unlike traceback frames added for exceptions seen in IR, we do this # even if there is no `traceback_name`. This is because the error will # have originated here and so we need it in the traceback. globals_static = emitter.static_name('globals', module_name) traceback_code = 'CPy_AddTraceback("%s", "%s", %d, %s);' % ( source_path.replace("\\", "\\\\"), fn.traceback_name or fn.name, fn.line, globals_static) # If fn is a method, then the first argument is a self param real_args = list(fn.args) if fn.class_name and not fn.decl.kind == FUNC_STATICMETHOD: arg = real_args.pop(0) emitter.emit_line('PyObject *obj_{} = self;'.format(arg.name)) # Need to order args as: required, optional, kwonly optional, kwonly required # This is because CPyArg_ParseTupleAndKeywords format string requires # them grouped in that way. groups = [[arg for arg in real_args if arg.kind == k] for k in range(ARG_NAMED_OPT + 1)] reordered_args = groups[ARG_POS] + groups[ARG_OPT] + groups[ARG_NAMED_OPT] + groups[ARG_NAMED] arg_names = ''.join('"{}", '.format(arg.name) for arg in reordered_args) emitter.emit_line('static char *kwlist[] = {{{}0}};'.format(arg_names)) for arg in real_args: emitter.emit_line('PyObject *obj_{}{};'.format( arg.name, ' = NULL' if arg.optional else '')) cleanups = ['CPy_DECREF(obj_{});'.format(arg.name) for arg in groups[ARG_STAR] + groups[ARG_STAR2]] arg_ptrs = [] # type: List[str] if groups[ARG_STAR] or groups[ARG_STAR2]: arg_ptrs += ['&obj_{}'.format(groups[ARG_STAR][0].name) if groups[ARG_STAR] else 'NULL'] arg_ptrs += ['&obj_{}'.format(groups[ARG_STAR2][0].name) if groups[ARG_STAR2] else 'NULL'] arg_ptrs += ['&obj_{}'.format(arg.name) for arg in reordered_args] emitter.emit_lines( 'if (!CPyArg_ParseTupleAndKeywords(args, kw, "{}", kwlist{})) {{'.format( make_format_string(fn.name, groups), ''.join(', ' + n for n in arg_ptrs)), 'return NULL;', '}') generate_wrapper_core(fn, emitter, groups[ARG_OPT] + groups[ARG_NAMED_OPT], cleanups=cleanups, traceback_code=traceback_code) emitter.emit_line('}')
def declare_finals( self, module: str, final_names: Iterable[Tuple[str, RType]], emitter: Emitter) -> None: for name, typ in final_names: static_name = emitter.static_name(name, module) emitter.context.declarations[static_name] = HeaderDeclaration( '{}{};'.format(emitter.ctype_spaced(typ), static_name), needs_export=True)
def declare_finals(self, module: str, final_names: Iterable[Tuple[str, RType]], emitter: Emitter) -> None: for name, typ in final_names: static_name = emitter.static_name(name, module) emitter.emit_line('extern {}{};'.format(emitter.ctype_spaced(typ), static_name))
def generate_globals_init(self, emitter: Emitter) -> None: emitter.emit_lines( '', 'int CPyGlobalsInit(void)', '{', 'static int is_initialized = 0;', 'if (is_initialized) return 0;', '' ) emitter.emit_line('CPy_Init();') for symbol, fixup in self.simple_inits: emitter.emit_line('{} = {};'.format(symbol, fixup)) for (_, literal), identifier in self.literals.items(): symbol = emitter.static_name(identifier, None) if isinstance(literal, int): actual_symbol = symbol symbol = INT_PREFIX + symbol emitter.emit_line( 'PyObject * {} = PyLong_FromString(\"{}\", NULL, 10);'.format( symbol, str(literal)) ) elif isinstance(literal, float): emitter.emit_line( '{} = PyFloat_FromDouble({});'.format(symbol, str(literal)) ) elif isinstance(literal, complex): emitter.emit_line( '{} = PyComplex_FromDoubles({}, {});'.format( symbol, str(literal.real), str(literal.imag)) ) elif isinstance(literal, str): emitter.emit_line( '{} = PyUnicode_FromStringAndSize({}, {});'.format( symbol, *encode_as_c_string(literal)) ) elif isinstance(literal, bytes): emitter.emit_line( '{} = PyBytes_FromStringAndSize({}, {});'.format( symbol, *encode_bytes_as_c_string(literal)) ) else: assert False, ('Literals must be integers, floating point numbers, or strings,', 'but the provided literal is of type {}'.format(type(literal))) emitter.emit_lines('if (unlikely({} == NULL))'.format(symbol), ' return -1;') # Ints have an unboxed representation. if isinstance(literal, int): emitter.emit_line( '{} = CPyTagged_FromObject({});'.format(actual_symbol, symbol) ) emitter.emit_lines( 'is_initialized = 1;', 'return 0;', '}', )
def declare_module(self, module_name: str, emitter: Emitter) -> None: # We declare two globals for each module: # one used internally in the implementation of module init to cache results # and prevent infinite recursion in import cycles, and one used # by other modules to refer to it. internal_static_name = self.module_internal_static_name(module_name, emitter) self.declare_global('CPyModule *', internal_static_name, initializer='NULL') static_name = emitter.static_name('module', module_name) self.declare_global('CPyModule *', static_name) self.simple_inits.append((static_name, 'Py_None'))
def final_definition( self, module: str, name: str, typ: RType, emitter: Emitter) -> str: static_name = emitter.static_name(name, module) # Here we rely on the fact that undefined value and error value are always the same if isinstance(typ, RTuple): # We need to inline because initializer must be static undefined = '{{ {} }}'.format(''.join(emitter.tuple_undefined_value_helper(typ))) else: undefined = emitter.c_undefined_value(typ) return '{}{} = {};'.format(emitter.ctype_spaced(typ), static_name, undefined)
def define_finals(self, final_names: Iterable[Tuple[str, RType]], emitter: Emitter) -> None: for name, typ in final_names: static_name = emitter.static_name(name, 'final') # Here we rely on the fact that undefined value and error value are always the same if isinstance(typ, RTuple): # We need to inline because initializer must be static undefined = '{{ {} }}'.format(''.join(emitter.tuple_undefined_value_helper(typ))) else: undefined = emitter.c_undefined_value(typ) emitter.emit_line('{}{} = {};'.format(emitter.ctype_spaced(typ), static_name, undefined))
def declare_static_pyobject(self, identifier: str, emitter: Emitter) -> None: symbol = emitter.static_name(identifier, None) self.declare_global('PyObject *', symbol)
def module_static_name(self, module_name: str, emitter: Emitter) -> str: return emitter.static_name('module', module_name)
def declare_internal_globals(self, module_name: str, emitter: Emitter) -> None: static_name = emitter.static_name('globals', module_name) self.declare_global('PyObject *', static_name)
def generate_module_def(self, emitter: Emitter, module_name: str, module: ModuleIR) -> None: """Emit the PyModuleDef struct for a module and the module init function.""" # Emit module methods module_prefix = emitter.names.private_name(module_name) emitter.emit_line( 'static PyMethodDef {}module_methods[] = {{'.format(module_prefix)) for fn in module.functions: if fn.class_name is not None or fn.name == TOP_LEVEL_NAME: continue emitter.emit_line(( '{{"{name}", (PyCFunction){prefix}{cname}, METH_VARARGS | METH_KEYWORDS, ' 'NULL /* docstring */}},').format(name=fn.name, cname=fn.cname( emitter.names), prefix=PREFIX)) emitter.emit_line('{NULL, NULL, 0, NULL}') emitter.emit_line('};') emitter.emit_line() # Emit module definition struct emitter.emit_lines( 'static struct PyModuleDef {}module = {{'.format(module_prefix), 'PyModuleDef_HEAD_INIT,', '"{}",'.format(module_name), 'NULL, /* docstring */', '-1, /* size of per-interpreter state of the module,', ' or -1 if the module keeps state in global variables. */', '{}module_methods'.format(module_prefix), '};') emitter.emit_line() # Emit module init function. If we are compiling just one module, this # will be the C API init function. If we are compiling 2+ modules, we # generate a shared library for the modules and shims that call into # the shared library, and in this case we use an internal module # initialized function that will be called by the shim. if len(self.modules) == 1: declaration = 'PyMODINIT_FUNC PyInit_{}(void)' else: declaration = 'PyObject *CPyInit_{}(void)' emitter.emit_lines(declaration.format(module_name), '{') module_static = self.module_static_name(module_name, emitter) emitter.emit_lines('if ({} != NULL) {{'.format(module_static), 'Py_INCREF({});'.format(module_static), 'return {};'.format(module_static), '}') emitter.emit_lines( '{} = PyModule_Create(&{}module);'.format(module_static, module_prefix), 'if ({} == NULL)'.format(module_static), ' return NULL;') module_globals = emitter.static_name('globals', module_name) emitter.emit_lines( '{} = PyModule_GetDict({});'.format(module_globals, module_static), 'if ({} == NULL)'.format(module_globals), ' return NULL;') self.generate_imports_init_section(module.imports, emitter) self.generate_from_imports_init_section( module_static, module.imports, module.from_imports, emitter, ) for cl in module.classes: type_struct = emitter.type_struct_name(cl) # FIXME: This ought to be driven by emitclass, maybe? if cl.traits: real_base = cl.real_base() bases = ([real_base] if real_base else []) + cl.traits emitter.emit_lines( '{}.tp_bases = PyTuple_Pack({}, {});'.format( type_struct, len(bases), ', '.join('&{}'.format(emitter.type_struct_name(b)) for b in bases))) emitter.emit_lines( 'if (PyType_Ready(&{}) < 0)'.format(type_struct), ' return NULL;') for (_, literal), identifier in module.literals.items(): symbol = emitter.static_name(identifier, None) if isinstance(literal, int): emitter.emit_lines( '{} = PyLong_FromString(\"{}\", NULL, 10);'.format( symbol, str(literal))) elif isinstance(literal, float): emitter.emit_lines('{} = PyFloat_FromDouble({});'.format( symbol, str(literal))) elif isinstance(literal, str): emitter.emit_lines( '{} = PyUnicode_FromStringAndSize({}, {});'.format( symbol, *encode_as_c_string(literal)), 'if ({} == NULL)'.format(symbol), ' return NULL;', ) elif isinstance(literal, bytes): emitter.emit_lines( '{} = PyBytes_FromStringAndSize({}, {});'.format( symbol, *encode_bytes_as_c_string(literal)), 'if ({} == NULL)'.format(symbol), ' return NULL;', ) else: assert False, ( 'Literals must be integers, floating point numbers, or strings,', 'but the provided literal is of type {}'.format( type(literal))) for cl in module.classes: name = cl.name type_struct = emitter.type_struct_name(cl) emitter.emit_lines( 'Py_INCREF(&{});'.format(type_struct), 'PyModule_AddObject({}, "{}", (PyObject *)&{});'.format( module_static, name, type_struct)) self.generate_top_level_call(module, emitter) emitter.emit_line('return {};'.format(module_static)) emitter.emit_line('}')
def generate_module_def(self, emitter: Emitter, module_name: str, module: ModuleIR) -> None: """Emit the PyModuleDef struct for a module and the module init function.""" # Emit module methods module_prefix = emitter.names.private_name(module_name) emitter.emit_line('static PyMethodDef {}module_methods[] = {{'.format(module_prefix)) for fn in module.functions: if fn.class_name is not None or fn.name == TOP_LEVEL_NAME: continue emitter.emit_line( ('{{"{name}", (PyCFunction){prefix}{cname}, METH_VARARGS | METH_KEYWORDS, ' 'NULL /* docstring */}},').format( name=fn.name, cname=fn.cname(emitter.names), prefix=PREFIX)) emitter.emit_line('{NULL, NULL, 0, NULL}') emitter.emit_line('};') emitter.emit_line() # Emit module definition struct emitter.emit_lines('static struct PyModuleDef {}module = {{'.format(module_prefix), 'PyModuleDef_HEAD_INIT,', '"{}",'.format(module_name), 'NULL, /* docstring */', '-1, /* size of per-interpreter state of the module,', ' or -1 if the module keeps state in global variables. */', '{}module_methods'.format(module_prefix), '};') emitter.emit_line() # Emit module init function. If we are compiling just one module, this # will be the C API init function. If we are compiling 2+ modules, we # generate a shared library for the modules and shims that call into # the shared library, and in this case we use an internal module # initialized function that will be called by the shim. if not self.use_shared_lib: declaration = 'PyMODINIT_FUNC PyInit_{}(void)'.format(module_name) else: declaration = 'PyObject *CPyInit_{}(void)'.format(exported_name(module_name)) emitter.emit_lines(declaration, '{') # Store the module reference in a static and return it when necessary. # This is separate from the *global* reference to the module that will # be populated when it is imported by a compiled module. We want that # reference to only be populated when the module has been succesfully # imported, whereas this we want to have to stop a circular import. module_static = self.module_internal_static_name(module_name, emitter) emitter.emit_lines('if ({}) {{'.format(module_static), 'Py_INCREF({});'.format(module_static), 'return {};'.format(module_static), '}') emitter.emit_lines('{} = PyModule_Create(&{}module);'.format(module_static, module_prefix), 'if (unlikely({} == NULL))'.format(module_static), ' return NULL;') emitter.emit_line( 'PyObject *modname = PyObject_GetAttrString((PyObject *){}, "__name__");'.format( module_static)) module_globals = emitter.static_name('globals', module_name) emitter.emit_lines('{} = PyModule_GetDict({});'.format(module_globals, module_static), 'if (unlikely({} == NULL))'.format(module_globals), ' return NULL;') # HACK: Manually instantiate generated classes here for cl in module.classes: if cl.is_generated: type_struct = emitter.type_struct_name(cl) emitter.emit_lines( '{t} = (PyTypeObject *)CPyType_FromTemplate({t}_template, NULL, modname);'. format(t=type_struct)) emitter.emit_lines('if (unlikely(!{}))'.format(type_struct), ' return NULL;') emitter.emit_lines('if (CPyGlobalsInit() < 0)', ' return NULL;') self.generate_top_level_call(module, emitter) emitter.emit_lines('Py_DECREF(modname);') emitter.emit_line('return {};'.format(module_static)) emitter.emit_line('}')
def module_internal_static_name(self, module_name: str, emitter: Emitter) -> str: return emitter.static_name(module_name + '_internal', None, prefix=MODULE_PREFIX)
def generate_c_for_modules(self) -> str: emitter = Emitter(self.context) module_irs = [module_ir for _, module_ir in self.modules] for module_name, module in self.modules: self.declare_module(module_name, emitter) self.declare_internal_globals(module_name, emitter) self.declare_imports(module.imports, emitter) for (_, literal), identifier in self.literals.items(): if isinstance(literal, int): symbol = emitter.static_name(identifier, None) self.declare_global('CPyTagged ', symbol) else: self.declare_static_pyobject(identifier, emitter) for module_name, module in self.modules: # Finals must be last (types can depend on declared above) self.declare_finals(module.final_names, emitter) for module in module_irs: for fn in module.functions: generate_function_declaration(fn, emitter) self.generate_literals(emitter) classes = [] for module_name, module in self.modules: classes.extend([(module_name, cl) for cl in module.classes]) # We must topo sort so that base classes are generated first. classes = sort_classes(classes) for module_name, cl in classes: generate_class_type_decl(cl, emitter) for module_name, cl in classes: generate_class(cl, module_name, emitter) emitter.emit_line() # Generate Python extension module definitions and module initialization functions. for module_name, module in self.modules: self.generate_module_def(emitter, module_name, module) for module_name, module in self.modules: for fn in module.functions: emitter.emit_line() generate_native_function(fn, emitter, self.source_paths[module_name], module_name) if fn.name != TOP_LEVEL_NAME: emitter.emit_line() generate_wrapper_function(fn, emitter) declarations = Emitter(self.context) declarations.emit_line('#include <Python.h>') declarations.emit_line('#include <CPy.h>') declarations.emit_line() for declaration in self.toposort_declarations(): declarations.emit_lines(*declaration.body) for static_def in self.context.statics.values(): declarations.emit_line(static_def) return ''.join(declarations.fragments + emitter.fragments)