def test_some_integer_type(): ffi = FFI() ffi.cdef(""" typedef int... foo_t; typedef unsigned long... bar_t; typedef struct { foo_t a, b; } mystruct_t; foo_t foobar(bar_t, mystruct_t); static const bar_t mu = -20; static const foo_t nu = 20; """) lib = verify(ffi, 'test_some_integer_type', """ typedef unsigned long long foo_t; typedef short bar_t; typedef struct { foo_t a, b; } mystruct_t; static foo_t foobar(bar_t x, mystruct_t s) { return (foo_t)x + s.a + s.b; } static const bar_t mu = -20; static const foo_t nu = 20; """) assert ffi.sizeof("foo_t") == ffi.sizeof("unsigned long long") assert ffi.sizeof("bar_t") == ffi.sizeof("short") maxulonglong = 2 ** 64 - 1 assert int(ffi.cast("foo_t", -1)) == maxulonglong assert int(ffi.cast("bar_t", -1)) == -1 assert lib.foobar(-1, [0, 0]) == maxulonglong assert lib.foobar(2 ** 15 - 1, [0, 0]) == 2 ** 15 - 1 assert lib.foobar(10, [20, 31]) == 61 assert lib.foobar(0, [0, maxulonglong]) == maxulonglong py.test.raises(OverflowError, lib.foobar, 2 ** 15, [0, 0]) py.test.raises(OverflowError, lib.foobar, -(2 ** 15) - 1, [0, 0]) py.test.raises(OverflowError, ffi.new, "mystruct_t *", [0, -1]) assert lib.mu == -20 assert lib.nu == 20
def init(): '''Initialize the module.''' with StackContext(Privilege.fileaccess): try: shutil.rmtree('container/standard/home') except FileNotFoundError: pass os.mkdir('container/standard/home', mode=0o771) try: shutil.rmtree('container/standard/cache') except FileNotFoundError: pass os.mkdir('container/standard/cache', mode=0o771) ffi = FFI() ffi.cdef('''int mount(const char source[], const char target[], const char filesystemtype[], unsigned long mountflags, const void *data);''') ffi.cdef('''int umount(const char *target);''') libc = ffi.dlopen('libc.so.6') with StackContext(Privilege.fullaccess): libc.umount(b'container/standard/dev') libc.mount(b'/dev', b'container/standard/dev', b'', MS_BIND, \ ffi.NULL) StdChal.null_fd = os.open('/dev/null', os.O_RDWR | os.O_CLOEXEC) StdChal.build_cache = {} StdChal.build_cache_refcount = {}
def solve_it(input_data): arr = [(int(n[0]), int(n[1])) for n in [line.split() for line in input_data.split("\n") if line]] nitems = arr[0][0] capacity = arr[0][1] values, weights = zip(*arr[1:]) ffi = FFI() ffi.cdef(""" typedef enum {false, true} bool; typedef struct { bool success; int value; bool *route; } Result; Result run(int *values, int *weights, int nitems, int capacity); """) fpath = "data/ks_4_0" lib = ffi.dlopen("libknap.so") res = lib.run(values, weights, nitems, capacity) out = "%d 1\n" % res.value out += " ".join([str(res.route[i]) for i in range(nitems)]) return out
def test_pipe(): ffi = FFI(backend=FakeBackend()) ffi.cdef("int pipe(int pipefd[2]);") C = ffi.dlopen(None) func = C.pipe assert func.name == 'pipe' assert func.BType == '<func (<pointer to <int>>), <int>, False>'
def build_ffi(include_recovery=False, include_schnorr=False, include_ecdh=False): ffi = FFI() source = "#include <secp256k1.h>" cdefs = definitions if include_recovery: cdefs += definitions_recovery source += "\n#include <secp256k1_recovery.h>" if include_schnorr: cdefs += definitions_schnorr source += "\n#include <secp256k1_schnorr.h>" if include_ecdh: cdefs += definitions_ecdh source += "\n#include <secp256k1_ecdh.h>" incpath = [os.environ['INCLUDE_DIR']] if 'INCLUDE_DIR' in os.environ else None libpath = [os.environ['LIB_DIR']] if 'LIB_DIR' in os.environ else None ffi.set_source( "_libsecp256k1", source, libraries=["secp256k1"], library_dirs=libpath, include_dirs=incpath) ffi.cdef(cdefs) return ffi
def test_vararg(): ffi = FFI(backend=FakeBackend()) ffi.cdef("short foo(int, ...);") C = ffi.dlopen(None) func = C.foo assert func.name == 'foo' assert func.BType == '<func (<int>), <short>, True>'
def test_no_args(): ffi = FFI(backend=FakeBackend()) ffi.cdef(""" int foo(void); """) C = ffi.dlopen(None) assert C.foo.BType == '<func (), <int>, False>'
def test_simple(): ffi = FFI(backend=FakeBackend()) ffi.cdef("double sin(double x);") m = ffi.dlopen(lib_m) func = m.sin # should be a callable on real backends assert func.name == 'sin' assert func.BType == '<func (<double>), <double>, False>'
def test_introspect_included_type(self): ffi1 = FFI() ffi2 = FFI() ffi1.cdef("typedef signed char schar_t; struct sint_t { int x; };") ffi2.include(ffi1) assert ffi1.list_types() == ffi2.list_types() == ( ['schar_t'], ['sint_t'], [])
def setup_class(cls): stack_ffi = FFI() stack_ffi.cdef(""" typedef intptr_t ptr_t; int vmp_binary_search_ranges(ptr_t ip, ptr_t * l, int count); int vmp_ignore_ip(ptr_t ip); int vmp_ignore_symbol_count(void); ptr_t * vmp_ignore_symbols(void); void vmp_set_ignore_symbols(ptr_t * symbols, int count); int vmp_read_vmaps(const char * fname); void vmp_native_disable(); """) with open("src/vmp_stack.c", "rb") as fd: source = fd.read().decode() libs = [] #['unwind', 'unwind-x86_64'] if sys.platform.startswith('linux'): libs = ['unwind', 'unwind-x86_64'] # trick: compile with _CFFI_USE_EMBEDDING=1 which will not define Py_LIMITED_API stack_ffi.set_source("vmprof.test._test_stack", source, include_dirs=['src'], define_macros=[('_CFFI_USE_EMBEDDING',1), ('PY_TEST',1), ('VMP_SUPPORTS_NATIVE_PROFILING',1)], libraries=libs, extra_compile_args=['-Werror', '-g']) stack_ffi.compile(verbose=True) from vmprof.test import _test_stack as clib cls.lib = clib.lib cls.ffi = clib.ffi
def _init_api(): lib_name = ctypes.util.find_library("ssh") if not lib_name: raise exceptions.PystasshException("libssh not found, please visit https://www.libssh.org/get-it/") ffi = FFI() lib = ffi.dlopen(lib_name) ffi.cdef(""" void* ssh_new(); int ssh_options_set(void*, int, char*); int ssh_connect(void*); int ssh_disconnect(void*); int ssh_is_connected(void*); char* ssh_get_error(void*); int ssh_userauth_password(void*, char*, char*); int ssh_userauth_autopubkey(void*, char*); void* ssh_channel_new(void*); int ssh_channel_open_session(void*); int ssh_channel_is_open(void*); void ssh_channel_free(void*); int ssh_channel_request_exec(void*, char*); int ssh_channel_request_pty(void*); int ssh_channel_request_shell(void*); int ssh_channel_get_exit_status(void*); int ssh_channel_read(void*, char*, int, int); int ssh_channel_send_eof(void*); """) return ffi, lib
def test_not_supported_bitfield_in_result(self): ffi = FFI(backend=self.Backend()) ffi.cdef("struct foo_s { int a,b,c,d,e; int x:1; };") e = py.test.raises(NotImplementedError, ffi.callback, "struct foo_s foo(void)", lambda: 42) assert str(e.value) == ("struct foo_s(*)(): " "callback with unsupported argument or return type or with '...'")
def load_inline_module(): """ Create an inline module, return the corresponding ffi and dll objects. """ from cffi import FFI # We can't rely on libc availability on Windows anymore, so we use our # own compiled wrappers (see https://bugs.python.org/issue23606). defs = """ double _numba_test_sin(double x); double _numba_test_cos(double x); int foo(int a, int b, int c); """ source = """ static int foo(int a, int b, int c) { return a + b * c; } """ ffi = FFI() ffi.cdef(defs) # Load the _helperlib namespace from numba import _helperlib return ffi, ffi.dlopen(_helperlib.__file__)
def _run_callback_in_thread(): ffi = FFI() ffi.cdef(""" typedef int (*mycallback_func_t)(int, int); int threaded_ballback_test(mycallback_func_t mycb); """) lib = ffi.verify(""" #include <pthread.h> typedef int (*mycallback_func_t)(int, int); void *my_wait_function(void *ptr) { mycallback_func_t cbfunc = (mycallback_func_t)ptr; cbfunc(10, 10); cbfunc(12, 15); return NULL; } int threaded_ballback_test(mycallback_func_t mycb) { pthread_t thread; pthread_create(&thread, NULL, my_wait_function, (void*)mycb); return 0; } """, extra_compile_args=['-pthread']) seen = [] @ffi.callback('int(*)(int,int)') def mycallback(x, y): time.sleep(0.022) seen.append((x, y)) return 0 lib.threaded_ballback_test(mycallback) count = 300 while len(seen) != 2: time.sleep(0.01) count -= 1 assert count > 0, "timeout" assert seen == [(10, 10), (12, 15)]
def test_redefine_common_type(): prefix = "" if sys.version_info < (3,) else "b" ffi = FFI() ffi.cdef("typedef char FILE;") assert repr(ffi.cast("FILE", 123)) == "<cdata 'char' %s'{'>" % prefix ffi.cdef("typedef char int32_t;") assert repr(ffi.cast("int32_t", 123)) == "<cdata 'char' %s'{'>" % prefix
def _setup_cffi(): class LazyLibrary(object): def __init__(self, ffi, libname): self._ffi = ffi self._libname = libname self._lib = None self._lock = threading.Lock() def __getattr__(self, name): if self._lib is None: with self._lock: if self._lib is None: self._lib = self._ffi.dlopen(self._libname) return getattr(self._lib, name) MODULES = ["libnvpair", "libzfs_core"] ffi = FFI() for module_name in MODULES: module = importlib.import_module("." + module_name, __package__) ffi.cdef(module.CDEF) lib = LazyLibrary(ffi, module.LIBRARY) setattr(module, "ffi", ffi) setattr(module, "lib", lib)
def test_function_pointer(self): ffi = FFI(backend=self.Backend()) def cb(charp): assert repr(charp).startswith("<cdata 'char *' 0x") return 42 fptr = ffi.callback("int(*)(const char *txt)", cb) assert fptr != ffi.callback("int(*)(const char *)", cb) assert repr(fptr) == "<cdata 'int(*)(char *)' calling %r>" % (cb,) res = fptr(b"Hello") assert res == 42 # if not sys.platform.startswith('linux'): py.test.skip("probably no symbol 'stderr' in the lib") ffi.cdef(""" int fputs(const char *, void *); void *stderr; """) ffi.C = ffi.dlopen(None) fptr = ffi.cast("int(*)(const char *txt, void *)", ffi.C.fputs) assert fptr == ffi.C.fputs assert repr(fptr).startswith("<cdata 'int(*)(char *, void *)' 0x") with FdWriteCapture() as fd: fptr(b"world\n", ffi.C.stderr) res = fd.getvalue() assert res == b'world\n'
def test_missing_function(self): ffi = FFI(backend=self.Backend()) ffi.cdef(""" int nonexistent(); """) m = ffi.dlopen(lib_m) assert not hasattr(m, 'nonexistent')
def main(filename): ffi = FFI() ffi.set_source( "pylibsize", """ # include "libsize.h" """, extra_objects=["libsize.so"], include_dirs=["."], ) ffi.cdef( """\ struct Size_return { int r0; /* w */ int r1; /* h */ }; extern struct Size_return Size(char* p0); """ ) ffi.compile(verbose=True, debug=True) from pylibsize import lib, ffi sizeinfo = lib.Size(filename.encode("utf-8")) print(sizeinfo, sizeinfo.r0, sizeinfo.r1)
def test_vararg(self): if not sys.platform.startswith('linux'): py.test.skip("probably no symbol 'stderr' in the lib") ffi = FFI(backend=self.Backend()) ffi.cdef(""" int fprintf(void *, const char *format, ...); void *stderr; """) ffi.C = ffi.dlopen(None) with FdWriteCapture() as fd: ffi.C.fprintf(ffi.C.stderr, b"hello with no arguments\n") ffi.C.fprintf(ffi.C.stderr, b"hello, %s!\n", ffi.new("char[]", b"world")) ffi.C.fprintf(ffi.C.stderr, ffi.new("char[]", b"hello, %s!\n"), ffi.new("char[]", b"world2")) ffi.C.fprintf(ffi.C.stderr, b"hello int %d long %ld long long %lld\n", ffi.cast("int", 42), ffi.cast("long", 84), ffi.cast("long long", 168)) ffi.C.fprintf(ffi.C.stderr, b"hello %p\n", ffi.NULL) res = fd.getvalue() assert res == (b"hello with no arguments\n" b"hello, world!\n" b"hello, world2!\n" b"hello int 42 long 84 long long 168\n" b"hello (nil)\n")
def test_unpack_args(): ffi = FFI() ffi.cdef("void foo0(void); void foo1(int); void foo2(int, int);") lib = verify(ffi, "test_unpack_args", """ void foo0(void) { } void foo1(int x) { } void foo2(int x, int y) { } """) assert 'foo0' in repr(lib.foo0) assert 'foo1' in repr(lib.foo1) assert 'foo2' in repr(lib.foo2) lib.foo0() lib.foo1(42) lib.foo2(43, 44) e1 = py.test.raises(TypeError, lib.foo0, 42) e2 = py.test.raises(TypeError, lib.foo0, 43, 44) e3 = py.test.raises(TypeError, lib.foo1) e4 = py.test.raises(TypeError, lib.foo1, 43, 44) e5 = py.test.raises(TypeError, lib.foo2) e6 = py.test.raises(TypeError, lib.foo2, 42) e7 = py.test.raises(TypeError, lib.foo2, 45, 46, 47) assert str(e1.value) == "foo0() takes no arguments (1 given)" assert str(e2.value) == "foo0() takes no arguments (2 given)" assert str(e3.value) == "foo1() takes exactly one argument (0 given)" assert str(e4.value) == "foo1() takes exactly one argument (2 given)" assert str(e5.value) == "foo2() takes exactly 2 arguments (0 given)" assert str(e6.value) == "foo2() takes exactly 2 arguments (1 given)" assert str(e7.value) == "foo2() takes exactly 2 arguments (3 given)"
def test_not_supported_bitfield_in_result(self): ffi = FFI(backend=self.Backend()) ffi.cdef("struct foo_s { int a,b,c,d,e; int x:1; };") e = py.test.raises(NotImplementedError, ffi.callback, "struct foo_s foo(void)", lambda: 42) assert str(e.value) == ("<struct foo_s(*)(void)>: " "cannot pass as argument or return value a struct with bit fields")
def test_unique_types(): CDEF = "struct foo_s; union foo_u; enum foo_e { AA };" ffi1 = FFI(); ffi1.cdef(CDEF); verify(ffi1, "test_unique_types_1", CDEF) ffi2 = FFI(); ffi2.cdef(CDEF); verify(ffi2, "test_unique_types_2", CDEF) # assert ffi1.typeof("char") is ffi2.typeof("char ") assert ffi1.typeof("long") is ffi2.typeof("signed long int") assert ffi1.typeof("double *") is ffi2.typeof("double*") assert ffi1.typeof("int ***") is ffi2.typeof(" int * * *") assert ffi1.typeof("int[]") is ffi2.typeof("signed int[]") assert ffi1.typeof("signed int*[17]") is ffi2.typeof("int *[17]") assert ffi1.typeof("void") is ffi2.typeof("void") assert ffi1.typeof("int(*)(int,int)") is ffi2.typeof("int(*)(int,int)") # # these depend on user-defined data, so should not be shared for name in ["struct foo_s", "union foo_u *", "enum foo_e", "struct foo_s *(*)()", "void(*)(struct foo_s *)", "struct foo_s *(*[5])[8]", ]: assert ffi1.typeof(name) is not ffi2.typeof(name) # sanity check: twice 'ffi1' assert ffi1.typeof("struct foo_s*") is ffi1.typeof("struct foo_s *")
def test_global_var_array(): ffi = FFI() ffi.cdef("int a[100];") lib = verify(ffi, 'test_global_var_array', 'int a[100] = { 9999 };') lib.a[42] = 123456 assert lib.a[42] == 123456 assert lib.a[0] == 9999
def test_verify_struct(): ffi = FFI() ffi.cdef("""struct foo_s { int b; short a; ...; }; struct bar_s { struct foo_s *f; };""") lib = verify(ffi, 'test_verify_struct', """struct foo_s { short a; int b; }; struct bar_s { struct foo_s *f; };""") ffi.typeof("struct bar_s *") p = ffi.new("struct foo_s *", {'a': -32768, 'b': -2147483648}) assert p.a == -32768 assert p.b == -2147483648 py.test.raises(OverflowError, "p.a -= 1") py.test.raises(OverflowError, "p.b -= 1") q = ffi.new("struct bar_s *", {'f': p}) assert q.f == p # assert ffi.offsetof("struct foo_s", "a") == 0 assert ffi.offsetof("struct foo_s", "b") == 4 assert ffi.offsetof(u+"struct foo_s", u+"b") == 4 # py.test.raises(TypeError, ffi.addressof, p) assert ffi.addressof(p[0]) == p assert ffi.typeof(ffi.addressof(p[0])) is ffi.typeof("struct foo_s *") assert ffi.typeof(ffi.addressof(p, "b")) is ffi.typeof("int *") assert ffi.addressof(p, "b")[0] == p.b
def netscape_spki_from_b64(b64): """Converts a base64 encoded Netscape SPKI DER to a crypto.NetscapeSPKI. PyOpenSSL does not yet support doing that by itself, so some work around through FFI and "internals-patching" trickery is required to perform this conversion. https://github.com/pyca/pyopenssl/issues/177 tracks the issue upstream. """ if not hasattr(netscape_spki_from_b64, 'NETSCAPE_SPKI_b64_decode'): from cffi import FFI as CFFI from OpenSSL._util import ffi as _sslffi, lib as _ssllib cffi = CFFI() cffi.cdef('void* NETSCAPE_SPKI_b64_decode(const char *str, int len);') lib = cffi.dlopen('libssl.so') def wrapper(b64, lib=lib): if isinstance(b64, str): b64 = b64.encode('ascii') b64_ptr = _sslffi.new('char[]', b64) spki_obj = lib.NETSCAPE_SPKI_b64_decode(b64_ptr, len(b64)) if spki_obj == cffi.NULL: raise ValueError("Invalid SPKI base64") def free(spki_obj, ref=b64_ptr): _ssllib.NETSCAPE_SPKI_free(spki_obj) return _sslffi.gc(spki_obj, free) netscape_spki_from_b64.func = wrapper ret = crypto.NetscapeSPKI() ret._spki = netscape_spki_from_b64.func(b64) return ret
def test_load_library(self): ffi = FFI() ffi.cdef("double sin(double x);") csrc = '/*hi there %s!3*/\n#include <math.h>\n' % self v = Verifier(ffi, csrc, force_generic_engine=self.generic) library = v.load_library() assert library.sin(12.3) == math.sin(12.3)
def test_void_star_accepts_string(self): ffi = FFI(backend=self.Backend()) ffi.cdef("""int strlen(const void *);""") needs_dlopen_none() lib = ffi.dlopen(None) res = lib.strlen(b"hello") assert res == 5
def test_address_of_global_var(): ffi = FFI() ffi.cdef(""" long bottom, bottoms[2]; long FetchRectBottom(void); long FetchRectBottoms1(void); #define FOOBAR 42 """) lib = verify(ffi, "test_address_of_global_var", """ long bottom, bottoms[2]; long FetchRectBottom(void) { return bottom; } long FetchRectBottoms1(void) { return bottoms[1]; } #define FOOBAR 42 """) lib.bottom = 300 assert lib.FetchRectBottom() == 300 lib.bottom += 1 assert lib.FetchRectBottom() == 301 lib.bottoms[1] = 500 assert lib.FetchRectBottoms1() == 500 lib.bottoms[1] += 2 assert lib.FetchRectBottoms1() == 502 # p = ffi.addressof(lib, 'bottom') assert ffi.typeof(p) == ffi.typeof("long *") assert p[0] == 301 p[0] += 1 assert lib.FetchRectBottom() == 302 p = ffi.addressof(lib, 'bottoms') assert ffi.typeof(p) == ffi.typeof("long(*)[2]") assert p[0] == lib.bottoms # py.test.raises(AttributeError, ffi.addressof, lib, 'unknown_var') py.test.raises(AttributeError, ffi.addressof, lib, "FOOBAR")
def test_verify_anonymous_struct_with_star_typedef(): ffi = FFI() ffi.cdef("typedef struct { int a; long b; } *foo_t;") verify(ffi, 'test_verify_anonymous_struct_with_star_typedef', "typedef struct { int a; long b; } *foo_t;") p = ffi.new("foo_t", {'b': 42}) assert p.b == 42
from cffi import FFI ffibuilder = FFI() ffibuilder.cdef(""" void evolve(double *u, double *u_previous, int nx, int ny, double a, double dt, double dx2, double dy2); """) ffibuilder.set_source("_evolve", # name of the output C extension """ void evolve(double *u, double *u_previous, int nx, int ny, double a, double dt, double dx2, double dy2); """, sources=['evolve.c'], extra_compile_args=['-O3'], library_dirs=['.'], # libraries=['evolve'], ) if __name__ == "__main__": ffibuilder.compile(verbose=True)
from cffi import FFI ffi = FFI() ffi.cdef(""" typedef void* piglet; piglet* piglet_create(void (*setup)(void), void (*draw)(void)); piglet* piglet_create_detail( void (*setup)(void), void (*draw)(void), unsigned x, unsigned y, unsigned width, unsigned height, unsigned fullscreen); void piglet_fullscreen(piglet* p, unsigned fullscreen); void piglet_set_rect(piglet* p, unsigned x, unsigned y, unsigned width, unsigned height); void piglet_destroy(piglet* p); """) lib = ffi.dlopen("../piglet.so"); def Create(setup, draw): return lib.piglet_create(ffi.callback("void(void)", setup), ffi.callback("void(void)", draw)) def Destroy(piglet): lib.piglet_destroy(piglet)
ffi.cdef(""" /* libpmem */ typedef int mode_t; const char *pmem_errormsg(void); void *pmem_map_file(const char *path, size_t len, int flags, mode_t mode, size_t *mapped_lenp, int *is_pmemp); int pmem_unmap(void *addr, size_t len); int pmem_has_hw_drain(void); int pmem_is_pmem(void *addr, size_t len); const char *pmem_check_version( unsigned major_required, unsigned minor_required); void pmem_persist(void *addr, size_t len); int pmem_msync(void *addr, size_t len); void pmem_flush(void *addr, size_t len); void pmem_drain(void); /* libpmemlog */ #define PMEMLOG_MIN_POOL ... typedef struct pmemlog PMEMlogpool; typedef int off_t; const char *pmemlog_errormsg(void); PMEMlogpool *pmemlog_open(const char *path); PMEMlogpool *pmemlog_create(const char *path, size_t poolsize, mode_t mode); void pmemlog_close(PMEMlogpool *plp); size_t pmemlog_nbyte(PMEMlogpool *plp); void pmemlog_rewind(PMEMlogpool *plp); off_t pmemlog_tell(PMEMlogpool *plp); int pmemlog_check(const char *path); int pmemlog_append(PMEMlogpool *plp, const void *buf, size_t count); const char *pmemlog_check_version( unsigned major_required, unsigned minor_required); void pmemlog_walk(PMEMlogpool *plp, size_t chunksize, int (*process_chunk)(const void *buf, size_t len, void *arg), void *arg); /* libpmemblk */ #define PMEMBLK_MIN_POOL ... #define PMEMBLK_MIN_BLK ... typedef struct pmemblk PMEMblkpool; const char *pmemblk_errormsg(void); PMEMblkpool *pmemblk_open(const char *path, size_t bsize); PMEMblkpool *pmemblk_create(const char *path, size_t bsize, size_t poolsize, mode_t mode); void pmemblk_close(PMEMblkpool *pbp); int pmemblk_check(const char *path, size_t bsize); size_t pmemblk_bsize(PMEMblkpool *pbp); size_t pmemblk_nblock(PMEMblkpool *pbp); int pmemblk_read(PMEMblkpool *pbp, void *buf, off_t blockno); int pmemblk_write(PMEMblkpool *pbp, const void *buf, off_t blockno); int pmemblk_set_zero(PMEMblkpool *pbp, off_t blockno); int pmemblk_set_error(PMEMblkpool *pbp, off_t blockno); const char *pmemblk_check_version( unsigned major_required, unsigned minor_required); /* libpmemobj */ typedef ... va_list; typedef struct pmemobjpool PMEMobjpool; #define PMEMOBJ_MIN_POOL ... #define PMEMOBJ_MAX_ALLOC_SIZE ... typedef struct pmemoid { uint64_t pool_uuid_lo; uint64_t off; } PMEMoid; static const PMEMoid OID_NULL; enum pobj_tx_stage { TX_STAGE_NONE, TX_STAGE_WORK, TX_STAGE_ONCOMMIT, TX_STAGE_ONABORT, TX_STAGE_FINALLY, ... }; const char *pmemobj_errormsg(void); PMEMobjpool *pmemobj_open(const char *path, const char *layout); PMEMobjpool *pmemobj_create(const char *path, const char *layout, size_t poolsize, mode_t mode); void pmemobj_close(PMEMobjpool *pop); int pmemobj_check(const char *path, const char *layout); PMEMoid pmemobj_root(PMEMobjpool *pop, size_t size); size_t pmemobj_root_size(PMEMobjpool *pop); void *pmemobj_direct(PMEMoid oid); int pmemobj_tx_begin(PMEMobjpool *pop, void *env, va_list *); void pmemobj_tx_abort(int errnum); void pmemobj_tx_commit(void); int pmemobj_tx_end(void); int pmemobj_tx_add_range(PMEMoid oid, uint64_t off, size_t size); int pmemobj_tx_add_range_direct(const void *ptr, size_t size); PMEMoid pmemobj_tx_alloc(size_t size, uint64_t type_num); PMEMoid pmemobj_tx_zalloc(size_t size, uint64_t type_num); PMEMoid pmemobj_tx_realloc(PMEMoid oid, size_t size, uint64_t type_num); PMEMoid pmemobj_tx_zrealloc(PMEMoid oid, size_t size, uint64_t type_num); PMEMoid pmemobj_tx_strdup(const char *s, uint64_t type_num); int pmemobj_tx_free(PMEMoid oid); enum pobj_tx_stage pmemobj_tx_stage(void); PMEMoid pmemobj_first(PMEMobjpool *pop); PMEMoid pmemobj_next(PMEMoid oid); uint64_t pmemobj_type_num(PMEMoid oid); """ + pmemobj_structs)
#!/usr/bin/env python3 from cffi import FFI ffibuilder = FFI() ffibuilder.cdef( r""" int parse_vlp_16(char [], float *, float *, float *, int); int parse_rs(char [], float *, float *, float *, int); int size_float(); """) ffibuilder.set_source("_parser", r""" int size_float() { return sizeof(float); } float unpack(char x[], int lend) { unsigned char a, b; a = x[0]; b = x[1]; if (lend == 1) { return a | (b << 8); } if (lend == 0) { return b | (a << 8); } } static int parse_vlp_16(char buf[], \ float *radius, float *intensity, float *azimuth, int x) {
from cffi import FFI ffi = FFI() ffi.cdef(""" #define FIONREAD ... """) ffi.set_source("trio_inotify._ioctl_c", """ #include <sys/ioctl.h> """, libraries=[]) if __name__ == "__main__": ffi.compile()
ffi.cdef(""" static inline uint64_t xor_murmur64(uint64_t h) ; static inline uint64_t xor_mix_split(uint64_t key, uint64_t seed) ; static inline uint64_t xor_rotl64(uint64_t n, unsigned int c) ; static inline uint32_t xor_reduce(uint32_t hash, uint32_t n) ; static inline uint64_t xor_fingerprint(uint64_t hash) ; static inline uint64_t xor_rng_splitmix64(uint64_t *seed) ; typedef struct xor8_s { uint64_t seed; uint64_t blockLength; uint8_t *fingerprints; // after xor8_allocate, will point to 3*blockLength values } xor8_t; struct xor_xorset_s { uint64_t xormask; uint32_t count; }; typedef struct xor16_s { uint64_t seed; uint64_t blockLength; uint16_t *fingerprints; // after xor16_allocate, will point to 3*blockLength values } xor16_t; typedef struct xor_xorset_s xor_xorset_t; struct xor_hashes_s { uint64_t h; uint32_t h0; uint32_t h1; uint32_t h2; }; typedef struct xor_hashes_s xor_hashes_t; struct xor_h0h1h2_s { uint32_t h0; uint32_t h1; uint32_t h2; }; typedef struct xor_h0h1h2_s xor_h0h1h2_t; struct xor_keyindex_s { uint64_t hash; uint32_t index; }; typedef struct xor_keyindex_s xor_keyindex_t; struct xor_setbuffer_s { xor_keyindex_t *buffer; uint32_t *counts; int insignificantbits; uint32_t slotsize; // should be 1<< insignificantbits uint32_t slotcount; size_t originalsize; }; typedef struct xor_setbuffer_s xor_setbuffer_t; static inline bool xor8_contain(uint64_t key, const xor8_t *filter); static inline bool xor16_contain(uint64_t key, const xor16_t *filter); static inline bool xor8_allocate(uint32_t size, xor8_t *filter); static inline bool xor16_allocate(uint32_t size, xor16_t *filter); static inline size_t xor8_size_in_bytes(const xor8_t *filter); static inline size_t xor16_size_in_bytes(const xor16_t *filter); static inline void xor8_free(xor8_t *filter); static inline void xor16_free(xor16_t *filter); static inline xor_hashes_t xor8_get_h0_h1_h2(uint64_t k, const xor8_t *filter) ; static inline xor_h0h1h2_t xor8_get_just_h0_h1_h2(uint64_t hash, const xor8_t *filter) ; static inline uint32_t xor8_get_h0(uint64_t hash, const xor8_t *filter) ; static inline uint32_t xor8_get_h1(uint64_t hash, const xor8_t *filter) ; static inline uint32_t xor8_get_h2(uint64_t hash, const xor8_t *filter) ; static inline uint32_t xor16_get_h0(uint64_t hash, const xor16_t *filter) ; static inline uint32_t xor16_get_h1(uint64_t hash, const xor16_t *filter) ; static inline uint32_t xor16_get_h2(uint64_t hash, const xor16_t *filter) ; static inline xor_hashes_t xor16_get_h0_h1_h2(uint64_t k, const xor16_t *filter) ; static inline bool xor_init_buffer(xor_setbuffer_t *buffer, size_t size) ; static inline void xor_free_buffer(xor_setbuffer_t *buffer) ; static inline void xor_buffered_increment_counter(uint32_t index, uint64_t hash, xor_setbuffer_t *buffer, xor_xorset_t *sets) ; static inline void xor_make_buffer_current(xor_setbuffer_t *buffer, xor_xorset_t *sets, uint32_t index, xor_keyindex_t *Q, size_t *Qsize) ; static inline void xor_buffered_decrement_counter(uint32_t index, uint64_t hash, xor_setbuffer_t *buffer, xor_xorset_t *sets, xor_keyindex_t *Q, size_t *Qsize) ; static inline void xor_flush_increment_buffer(xor_setbuffer_t *buffer, xor_xorset_t *sets) ; static inline void xor_flush_decrement_buffer(xor_setbuffer_t *buffer, xor_xorset_t *sets, xor_keyindex_t *Q, size_t *Qsize) ; static inline uint32_t xor_flushone_decrement_buffer(xor_setbuffer_t *buffer, xor_xorset_t *sets, xor_keyindex_t *Q, size_t *Qsize) ; bool xor8_buffered_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) ; bool xor8_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) ; bool xor16_buffered_populate(const uint64_t *keys, uint32_t size, xor16_t *filter) ; bool xor16_populate(const uint64_t *keys, uint32_t size, xor16_t *filter) ; """)
from timeit import Timer from cffi import FFI ffi = FFI() if __name__ == "__main__": with open("hellomodule.h") as header: ffi.cdef(header.read()) lib = ffi.dlopen("fib.so") s = lib.fib(47) print([s[i] for i in range(47)]) t = Timer(lambda: lib.fib(47)) print(t.timeit())
typedef unsigned int pid_t; GArray * g_array_new (gboolean zero_terminated, gboolean clear_, guint element_size); GArray * g_array_append_vals(GArray* a, gconstpointer v, guint len); ''' + ''.join(altered_lines) cdef_string = cdef_string[:-2] # Remove the trailing } cdef_string += ''' extern "Python" void onDCCallback(int pid, void* socket, cb_event_loop* cb_loop); extern "Python" void onIceCallback(IceCandidate_C ice); extern "Python" void onOpened(void); extern "Python" void onClosed(int pid); extern "Python" void onStringMsg(int pid, const char* message); extern "Python" void onBinaryMsg(void* message); extern "Python" void onError(const char* description); ''' ffibuilder.cdef(cdef_string) if __name__ == "__main__": ffibuilder.compile(verbose=True)
ffi.cdef(""" typedef void * (*yajl_malloc_func)(void *ctx, size_t sz); typedef void (*yajl_free_func)(void *ctx, void * ptr); typedef void * (*yajl_realloc_func)(void *ctx, void * ptr, size_t sz); typedef struct { yajl_malloc_func malloc; yajl_realloc_func realloc; yajl_free_func free; void * ctx; } yajl_alloc_funcs; typedef struct yajl_handle_t * yajl_handle; typedef enum { yajl_status_ok, yajl_status_client_canceled, yajl_status_error } yajl_status; typedef enum { yajl_allow_comments = 0x01, yajl_dont_validate_strings = 0x02, yajl_allow_trailing_garbage = 0x04, yajl_allow_multiple_values = 0x08, yajl_allow_partial_values = 0x10 } yajl_option; typedef struct { int (* yajl_null)(void * ctx); int (* yajl_boolean)(void * ctx, int boolVal); int (* yajl_integer)(void * ctx, long long integerVal); int (* yajl_double)(void * ctx, double doubleVal); int (* yajl_number)(void * ctx, const char * numberVal, size_t numberLen); int (* yajl_string)(void * ctx, const unsigned char * stringVal, size_t stringLen); int (* yajl_start_map)(void * ctx); int (* yajl_map_key)(void * ctx, const unsigned char * key, size_t stringLen); int (* yajl_end_map)(void * ctx); int (* yajl_start_array)(void * ctx); int (* yajl_end_array)(void * ctx); } yajl_callbacks; int yajl_version(void); yajl_handle yajl_alloc(const yajl_callbacks *callbacks, yajl_alloc_funcs *afs, void *ctx); int yajl_config(yajl_handle h, yajl_option opt, ...); yajl_status yajl_parse(yajl_handle hand, const unsigned char *jsonText, size_t jsonTextLength); yajl_status yajl_complete_parse(yajl_handle hand); unsigned char* yajl_get_error(yajl_handle hand, int verbose, const unsigned char *jsonText, size_t jsonTextLength); void yajl_free_error(yajl_handle hand, unsigned char * str); void yajl_free(yajl_handle handle); """)
import os from cffi import FFI ffi = FFI() ffi.cdef( """double calc_distance_c(double*, double*, int, double*, double*, int, double, double, double, double, double);""") base_dir = os.path.dirname(os.path.abspath(__file__)), base_dir = os.path.join(base_dir[0], "..") for file in os.listdir(base_dir): print(file) if file.startswith("MasSpOTCppToPy") and file.endswith(".so"): so_path = os.path.join(base_dir, file) lib = ffi.dlopen(so_path) # Above searches for .so files in installed package. def spectral_distance(spec1, spec2, lam=1, eps=0.1, tol=1e-05, threshold=1e+02, max_iter=500): mzs1, ints1 = zip(*[x for x in spec1.confs if x[1] > 0]) mzs2, ints2 = zip(*[x for x in spec2.confs if x[1] > 0])
def _load_header(path: str) -> str: """ Load a C header file and convert it to something parseable by cffi. """ data = pkgutil.get_data(__name__, path) if data is None: # pragma: no cover raise RuntimeError("Couldn't load 'glean.h'") data_str = data.decode("utf-8") return "\n".join( line for line in data_str.splitlines() if not line.startswith("#include") ) ffi = FFI() ffi.cdef(_load_header("glean.h")) lib = ffi.dlopen(str(Path(__file__).parent / get_shared_object_filename())) lib.glean_enable_logging() def make_config( data_dir: Path, package_name: str, upload_enabled: bool, max_events: int, ) -> Any: """ Make an `FfiConfiguration` object. Args: data_dir (pathlib.Path): Path to the Glean data directory. package_name (str): The name of the package to report in Glean's pings. """ data_dir = ffi.new("char[]", ffi_encode_string(str(data_dir)))
from cffi import FFI root = os.path.dirname(__file__) print(root) kwdfile = 'c:/workspace/pyfvs2/data/pnt01.key' variant = 'pn' #libname = '{root}/bin/libFVS_{variant}.dll'.format(**locals())' libname = 'libFVS_pn' ffi = FFI() fvs = ffi.dlopen(libname) ffi.cdef(""" void fvs_(int* rtn); """) ffi.cdef(""" void fvsSetCmdLine_(char *cmdline, int lencl, int* rtn); """) s = '--keywordfile={}'.format(kwdfile) cmdline = bytes(s.encode()) _cmdline = ffi.new('char[]', cmdline) _lencl = ffi.new('int', len(cmdline)) _rtn = ffi.new('int* rtn', 0) fvs.fvsSetCmdLine_(_cmdline, _lencl, _rtn) fvs.fvs_(rtn)
class libperipheryled: def __init__(self): self.ffi = FFI() # Specify each C function, struct and constant you want a Python binding for # Copy-n-paste with minor edits self.ffi.cdef(""" enum led_error_code { LED_ERROR_ARG = -1, LED_ERROR_OPEN = -2, LED_ERROR_QUERY = -3, LED_ERROR_IO = -4, LED_ERROR_CLOSE = -5, }; struct led_handle { char name[64]; unsigned int max_brightness; struct { int c_errno; char errmsg[96]; } error; }; typedef struct led_handle led_t; led_t *led_new(void); int led_open(led_t *led, const char *name); int led_read(led_t *led, bool *value); int led_write(led_t *led, bool value); int led_close(led_t *led); void led_free(led_t *led); int led_get_brightness(led_t *led, unsigned int *brightness); int led_get_max_brightness(led_t *led, unsigned int *max_brightness); int led_set_brightness(led_t *led, unsigned int brightness); int led_name(led_t *led, char *str, size_t len); int led_tostring(led_t *led, char *str, size_t len); int led_errno(led_t *led); const char *led_errmsg(led_t *led); """) self.lib = self.ffi.dlopen("/usr/local/lib/libperipheryled.so") def open(self, device): """Open LED device and return handle. """ handle = self.lib.led_new() if self.lib.led_open(handle, device.encode('utf-8')) < 0: raise RuntimeError( self.ffi.string(self.lib.led_errmsg(handle)).decode('utf-8')) return handle def close(self, handle): """Close I2C device. """ if self.lib.led_close(handle) < 0: raise RuntimeError( self.ffi.string(self.lib.led_errmsg(handle)).decode('utf-8')) else: self.lib.led_free(handle) def setBrightness(self, handle, value): """Write value to led brightness register. """ # Transfer a transaction with one I2C message if self.lib.led_set_brightness(handle, value) < 0: raise RuntimeError( self.ffi.string(self.lib.led_errmsg(handle)).decode('utf-8'))
('LIBUV_EMBED', int(LIBUV_EMBED)), ] def _define_macro(name, value): LIBUV_MACROS.append((name, value)) if sys.platform != 'win32': _define_macro('_LARGEFILE_SOURCE', 1) _define_macro('_FILE_OFFSET_BITS', 64) if sys.platform.startswith('linux'): _add_library('dl') _add_library('rt') _define_macro('_GNU_SOURCE', 1) _define_macro('_POSIX_C_SOURCE', '200112') ffibuilder.cdef(data) ffibuilder.set_source("_fatuv", _source, extra_compile_args=['-g'], sources=LIBUV_SOURCES, depends=LIBUV_SOURCES, include_dirs=LIBUV_INCLUDE_DIRS, libraries=list(LIBUV_LIBRARIES), define_macros=list(LIBUV_MACROS)) if __name__ == "__main__": ffibuilder.compile(verbose=True)
def load_ool_module(): """ Compile an out-of-line module, return the corresponding ffi and module objects. """ from cffi import FFI numba_complex = """ typedef struct _numba_complex { double real; double imag; } numba_complex; """ defs = numba_complex + """ double sin(double x); double cos(double x); int foo(int a, int b, int c); void vsSin(int n, float* x, float* y); void vdSin(int n, double* x, double* y); void vector_real(numba_complex *c, double *real, int n); void vector_imag(numba_complex *c, double *imag, int n); """ source = numba_complex + """ static int foo(int a, int b, int c) { return a + b * c; } void vsSin(int n, float* x, float* y) { int i; for (i=0; i<n; i++) y[i] = sin(x[i]); } void vdSin(int n, double* x, double* y) { int i; for (i=0; i<n; i++) y[i] = sin(x[i]); } static void vector_real(numba_complex *c, double *real, int n) { int i; for (i = 0; i < n; i++) real[i] = c[i].real; } static void vector_imag(numba_complex *c, double *imag, int n) { int i; for (i = 0; i < n; i++) imag[i] = c[i].imag; } """ ffi = FFI() ffi.set_source('cffi_usecases_ool', source) ffi.cdef(defs, override=True) tmpdir = temp_directory('test_cffi') ffi.compile(tmpdir=tmpdir) sys.path.append(tmpdir) try: mod = import_dynamic('cffi_usecases_ool') cffi_support.register_module(mod) cffi_support.register_type(mod.ffi.typeof('struct _numba_complex'), complex128) return mod.ffi, mod finally: sys.path.remove(tmpdir)
ffi.cdef(""" typedef size_t mpd_size_t; /* unsigned size type */ typedef ssize_t mpd_ssize_t; /* signed size type */ typedef size_t mpd_uint_t; #define MPD_SIZE_MAX ... #define MPD_SSIZE_MIN ... #define MPD_SSIZE_MAX ... const char *mpd_version(void); void mpd_free(void *ptr); typedef struct mpd_context_t { mpd_ssize_t prec; /* precision */ mpd_ssize_t emax; /* max positive exp */ mpd_ssize_t emin; /* min negative exp */ uint32_t traps; /* status events that should be trapped */ uint32_t status; /* status flags */ uint32_t newtrap; /* set by mpd_addstatus_raise() */ int round; /* rounding mode */ int clamp; /* clamp mode */ int allcr; /* all functions correctly rounded */ } mpd_context_t; enum { MPD_ROUND_UP, /* round away from 0 */ MPD_ROUND_DOWN, /* round toward 0 (truncate) */ MPD_ROUND_CEILING, /* round toward +infinity */ MPD_ROUND_FLOOR, /* round toward -infinity */ MPD_ROUND_HALF_UP, /* 0.5 is rounded up */ MPD_ROUND_HALF_DOWN, /* 0.5 is rounded down */ MPD_ROUND_HALF_EVEN, /* 0.5 is rounded to even */ MPD_ROUND_05UP, /* round zero or five away from 0 */ MPD_ROUND_TRUNC, /* truncate, but set infinity */ MPD_ROUND_GUARD }; #define MPD_Clamped ... #define MPD_Conversion_syntax ... #define MPD_Division_by_zero ... #define MPD_Division_impossible ... #define MPD_Division_undefined ... #define MPD_Float_operation ... #define MPD_Fpu_error ... #define MPD_Inexact ... #define MPD_Invalid_context ... #define MPD_Invalid_operation ... #define MPD_Malloc_error ... #define MPD_Not_implemented ... #define MPD_Overflow ... #define MPD_Rounded ... #define MPD_Subnormal ... #define MPD_Underflow ... #define MPD_Max_status ... /* Conditions that result in an IEEE 754 exception */ #define MPD_IEEE_Invalid_operation ... /* Errors that require the result of an operation to be set to NaN */ #define MPD_Errors ... void mpd_maxcontext(mpd_context_t *ctx); int mpd_qsetprec(mpd_context_t *ctx, mpd_ssize_t prec); int mpd_qsetemax(mpd_context_t *ctx, mpd_ssize_t emax); int mpd_qsetemin(mpd_context_t *ctx, mpd_ssize_t emin); int mpd_qsetround(mpd_context_t *ctx, int newround); int mpd_qsettraps(mpd_context_t *ctx, uint32_t flags); int mpd_qsetstatus(mpd_context_t *ctx, uint32_t flags); int mpd_qsetclamp(mpd_context_t *ctx, int c); typedef struct mpd_t { uint8_t flags; mpd_ssize_t exp; mpd_ssize_t digits; mpd_ssize_t len; mpd_ssize_t alloc; mpd_uint_t *data; } mpd_t; #define MPD_POS ... #define MPD_NEG ... #define MPD_INF ... #define MPD_NAN ... #define MPD_SNAN ... #define MPD_SPECIAL ... #define MPD_STATIC ... #define MPD_STATIC_DATA ... #define MPD_SHARED_DATA ... #define MPD_CONST_DATA ... #define MPD_DATAFLAGS ... mpd_t *mpd_qnew(void); void mpd_del(mpd_t *dec); /* Operations */ void mpd_qabs(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); void mpd_qplus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); void mpd_qminus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); void mpd_qsqrt(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); void mpd_qexp(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); void mpd_qln(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); void mpd_qlog10(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); void mpd_qlogb(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); void mpd_qinvert(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); void mpd_qmax(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qmax_mag(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qmin(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qmin_mag(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qadd(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qsub(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qmul(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qdiv(mpd_t *q, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qdivint(mpd_t *q, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qfma(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_t *c, const mpd_context_t *ctx, uint32_t *status); void mpd_qrem(mpd_t *r, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qrem_near(mpd_t *r, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qpow(mpd_t *result, const mpd_t *base, const mpd_t *exp, const mpd_context_t *ctx, uint32_t *status); void mpd_qpowmod(mpd_t *result, const mpd_t *base, const mpd_t *exp, const mpd_t *mod, const mpd_context_t *ctx, uint32_t *status); int mpd_qcopy_sign(mpd_t *result, const mpd_t *a, const mpd_t *b, uint32_t *status); int mpd_qcopy_abs(mpd_t *result, const mpd_t *a, uint32_t *status); int mpd_qcopy_negate(mpd_t *result, const mpd_t *a, uint32_t *status); void mpd_qdivmod(mpd_t *q, mpd_t *r, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qand(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qor(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qxor(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); int mpd_same_quantum(const mpd_t *a, const mpd_t *b); void mpd_qround_to_intx(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); void mpd_qround_to_int(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); int mpd_qcopy(mpd_t *result, const mpd_t *a, uint32_t *status); int mpd_qcmp(const mpd_t *a, const mpd_t *b, uint32_t *status); int mpd_qcompare(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); int mpd_qcompare_signal(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); int mpd_compare_total(mpd_t *result, const mpd_t *a, const mpd_t *b); int mpd_compare_total_mag(mpd_t *result, const mpd_t *a, const mpd_t *b); void mpd_qnext_toward(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qnext_minus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); void mpd_qnext_plus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); void mpd_qquantize(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qrotate(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qscaleb(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qshift(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); void mpd_qreduce(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); /* Get attributes */ uint8_t mpd_sign(const mpd_t *dec); int mpd_isnegative(const mpd_t *dec); int mpd_ispositive(const mpd_t *dec); int mpd_iszero(const mpd_t *dec); int mpd_isfinite(const mpd_t *dec); int mpd_isinfinite(const mpd_t *dec); int mpd_issigned(const mpd_t *dec); int mpd_isnan(const mpd_t *dec); int mpd_issnan(const mpd_t *dec); int mpd_isspecial(const mpd_t *dec); int mpd_isqnan(const mpd_t *dec); int mpd_isnormal(const mpd_t *dec, const mpd_context_t *ctx); int mpd_issubnormal(const mpd_t *dec, const mpd_context_t *ctx); mpd_ssize_t mpd_adjexp(const mpd_t *dec); mpd_ssize_t mpd_etiny(const mpd_context_t *ctx); mpd_ssize_t mpd_etop(const mpd_context_t *ctx); mpd_t *mpd_qncopy(const mpd_t *a); /* Set attributes */ void mpd_set_sign(mpd_t *result, uint8_t sign); void mpd_set_positive(mpd_t *result); void mpd_clear_flags(mpd_t *result); void mpd_seterror(mpd_t *result, uint32_t flags, uint32_t *status); void mpd_setspecial(mpd_t *dec, uint8_t sign, uint8_t type); /* I/O */ void mpd_qimport_u16(mpd_t *result, const uint16_t *srcdata, size_t srclen, uint8_t srcsign, uint32_t srcbase, const mpd_context_t *ctx, uint32_t *status); size_t mpd_qexport_u16(uint16_t **rdata, size_t rlen, uint32_t base, const mpd_t *src, uint32_t *status); void mpd_qset_string(mpd_t *dec, const char *s, const mpd_context_t *ctx, uint32_t *status); void mpd_qset_uint(mpd_t *result, mpd_uint_t a, const mpd_context_t *ctx, uint32_t *status); void mpd_qset_ssize(mpd_t *result, mpd_ssize_t a, const mpd_context_t *ctx, uint32_t *status); void mpd_qsset_ssize(mpd_t *result, mpd_ssize_t a, const mpd_context_t *ctx, uint32_t *status); mpd_ssize_t mpd_qget_ssize(const mpd_t *dec, uint32_t *status); int mpd_lsnprint_signals(char *dest, int nmemb, uint32_t flags, const char *signal_string[]); #define MPD_MAX_SIGNAL_LIST ... const char *dec_signal_string[]; void mpd_qfinalize(mpd_t *result, const mpd_context_t *ctx, uint32_t *status); const char *mpd_class(const mpd_t *a, const mpd_context_t *ctx); /* format specification */ typedef struct mpd_spec_t { mpd_ssize_t min_width; /* minimum field width */ mpd_ssize_t prec; /* fraction digits or significant digits */ char type; /* conversion specifier */ char align; /* alignment */ char sign; /* sign printing/alignment */ char fill[5]; /* fill character */ const char *dot; /* decimal point */ const char *sep; /* thousands separator */ const char *grouping; /* grouping of digits */ } mpd_spec_t; char *mpd_to_sci(const mpd_t *dec, int fmt); char *mpd_to_eng(const mpd_t *dec, int fmt); int mpd_parse_fmt_str(mpd_spec_t *spec, const char *fmt, int caps); int mpd_validate_lconv(mpd_spec_t *spec); char *mpd_qformat_spec(const mpd_t *dec, const mpd_spec_t *spec, const mpd_context_t *ctx, uint32_t *status); """)
include_dirs=[src_dir, htslib_dir], sources=[ os.path.join(src_dir, x) for x in ('levenshtein.c', 'medaka_bamiter.c', 'medaka_common.c', 'medaka_khcounter.c', 'clair3_pileup.c', 'clair3_full_alignment.c') ], extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, extra_objects=['libhts.a']) cdef = [ "typedef struct { ...; } bam_fset;" "bam_fset* create_bam_fset(char* fname);" "void destroy_bam_fset(bam_fset* fset);" ] for header in ('clair3_pileup.h', 'clair3_full_alignment.h'): with open(os.path.join(src_dir, header), 'r') as fh: # remove directives lines = ''.join(x for x in fh.readlines() if not x.startswith('#')) cdef.append(lines) ffibuilder.cdef('\n\n'.join(cdef)) if __name__ == "__main__": ffibuilder.compile(verbose=True) run("cp {}/libclair3*.so {}/libclair3.so".format(file_directory, file_directory), shell=True)
import glob import os from cffi import FFI # include_dirs = [os.path.join('libraries', 'Rmath', 'src'), # os.path.join('libraries', 'Rmath', 'include')] # rmath_src = glob.glob(os.path.join('libraries', 'Rmath', 'src', '*.c')) ffi = FFI() ffi.set_source('_khash_ffi', '#include "khash_int2int.h"') ffi.cdef('''\ typedef int... khint64_t; static inline void *khash_int2int_init(void); static void khash_int2int_destroy(void *); static inline khint64_t khash_int2int_get(void *, khint64_t, khint64_t); static inline int khash_int2int_set(void *, khint64_t, khint64_t); ''') if __name__ == '__main__': ffi.compile(verbose=True)
] c_header_source = r''' #include <xenctrl.h> #include <xencall.h> #include <libxl.h> #include <xendevicemodel.h> #include <xenevtchn.h> #include <xenforeignmemory.h> #include <xenguest.h> #include <xenstore.h> #include <xen/domctl.h> #include <xen/vm_event.h> #include <xen/io/ring.h> #include <xen/sys/privcmd.h> #include <sys/mman.h> #define FOO__HYPERVISOR_domctl __HYPERVISOR_domctl ''' #include <xentoollog.h> cdef = read_local_file('cdef.h') ffi = FFI() ffi.set_source('xendbg.xen._bindings', c_header_source, libraries=libraries) ffi.cdef(cdef) if __name__ == '__main__': ffi.compile(verbose=True)
from cffi import FFI ffi = FFI() ffi.cdef(""" void OutputDebugStringW(const wchar_t*); """) kernel32 = ffi.dlopen('kernel32.dll') class OutputDebugStringWriter: def write(self, value): kernel32.OutputDebugStringW(value.rstrip()) def flush(self): pass
of work, the better your chances are of landing in the main chain. Feel free to modify this code in any way, or reimplement it in a different language or on specialized hardware. Good luck! """ def flatten(l): return [x for lp in l for x in lp] ffi = FFI() ffi.cdef( "int go(uint64_t, uint64_t, const uint8_t*, const uint8_t*, const uint8_t*, const uint8_t*, uint32_t, uint64_t*);" ) lib = ffi.dlopen("aesniminer-kernel/target/release/libaesniminer_kernel.so") # cuda_mod = SourceModule(open("kernel2.cu", "rb").read()) # go = cuda_mod.get_function('go') # test_AES = cuda_mod.get_function('test_AES') # key = '\0'*32 # expanded = numpy.array([x % (2**32) for x in flatten(pyaes.aes.AES(key)._Ke)], dtype=numpy.uint32) # # text = ''.join(chr(i) for i in range(16)) # text = '\0' * 16 # text_arr = numpy.array(map(ord, text), dtype=np.uint8) # test_AES(drv.In(expanded), drv.InOut(text_arr), block=(1, 1, 1)) # asdf = AES.new(key)
__darknetffi__.cdef(""" typedef struct network network; typedef struct layer layer; typedef struct{ int *leaf; int n; int *parent; int *child; int *group; char **name; int groups; int *group_size; int *group_offset; } tree; typedef enum{ LOGISTIC, RELU, RELIE, LINEAR, RAMP, TANH, PLSE, LEAKY, ELU, LOGGY, STAIR, HARDTAN, LHTAN } ACTIVATION; typedef enum { CONVOLUTIONAL, DECONVOLUTIONAL, CONNECTED, MAXPOOL, SOFTMAX, DETECTION, DROPOUT, CROP, ROUTE, COST, NORMALIZATION, AVGPOOL, LOCAL, SHORTCUT, ACTIVE, RNN, GRU, LSTM, CRNN, BATCHNORM, NETWORK, XNOR, REGION, YOLO, REORG, UPSAMPLE, LOGXENT, L2NORM, BLANK } LAYERTYPE; typedef enum{ SSE, MASKED, L1, SEG, SMOOTH, WGAN } COSTTYPE; struct layer{ LAYERTYPE type; ACTIVATION activation; COSTTYPE cost_type; void (*forward); void (*backward); void (*update); void (*forward_gpu); void (*backward_gpu); void (*update_gpu); int batch_normalize; int shortcut; int batch; int forced; int flipped; int inputs; int outputs; int nweights; int nbiases; int extra; int truths; int h,w,c; int out_h, out_w, out_c; int n; int max_boxes; int groups; int size; int side; int stride; int reverse; int flatten; int spatial; int pad; int sqrt; int flip; int index; int binary; int xnor; int steps; int hidden; int truth; float smooth; float dot; float angle; float jitter; float saturation; float exposure; float shift; float ratio; float learning_rate_scale; float clip; int softmax; int classes; int coords; int background; int rescore; int objectness; int joint; int noadjust; int reorg; int log; int tanh; int *mask; int total; float alpha; float beta; float kappa; float coord_scale; float object_scale; float noobject_scale; float mask_scale; float class_scale; int bias_match; int random; float ignore_thresh; float truth_thresh; float thresh; float focus; int classfix; int absolute; int onlyforward; int stopbackward; int dontload; int dontsave; int dontloadscales; float temperature; float probability; float scale; char * cweights; int * indexes; int * input_layers; int * input_sizes; int * map; float * rand; float * cost; float * state; float * prev_state; float * forgot_state; float * forgot_delta; float * state_delta; float * combine_cpu; float * combine_delta_cpu; float * concat; float * concat_delta; float * binary_weights; float * biases; float * bias_updates; float * scales; float * scale_updates; float * weights; float * weight_updates; float * delta; float * output; float * loss; float * squared; float * norms; float * spatial_mean; float * mean; float * variance; float * mean_delta; float * variance_delta; float * rolling_mean; float * rolling_variance; float * x; float * x_norm; float * m; float * v; float * bias_m; float * bias_v; float * scale_m; float * scale_v; float *z_cpu; float *r_cpu; float *h_cpu; float * prev_state_cpu; float *temp_cpu; float *temp2_cpu; float *temp3_cpu; float *dh_cpu; float *hh_cpu; float *prev_cell_cpu; float *cell_cpu; float *f_cpu; float *i_cpu; float *g_cpu; float *o_cpu; float *c_cpu; float *dc_cpu; float * binary_input; struct layer *input_layer; struct layer *self_layer; struct layer *output_layer; struct layer *reset_layer; struct layer *update_layer; struct layer *state_layer; struct layer *input_gate_layer; struct layer *state_gate_layer; struct layer *input_save_layer; struct layer *state_save_layer; struct layer *input_state_layer; struct layer *state_state_layer; struct layer *input_z_layer; struct layer *state_z_layer; struct layer *input_r_layer; struct layer *state_r_layer; struct layer *input_h_layer; struct layer *state_h_layer; struct layer *wz; struct layer *uz; struct layer *wr; struct layer *ur; struct layer *wh; struct layer *uh; struct layer *uo; struct layer *wo; struct layer *uf; struct layer *wf; struct layer *ui; struct layer *wi; struct layer *ug; struct layer *wg; tree *softmax_tree; size_t workspace_size; }; typedef enum { CONSTANT, STEP, EXP, POLY, STEPS, SIG, RANDOM } LEARNINGRATEPOLICY; typedef struct network{ int n; int batch; size_t *seen; int *t; float epoch; int subdivisions; layer *layers; float *output; LEARNINGRATEPOLICY policy; float learning_rate; float momentum; float decay; float gamma; float scale; float power; int time_steps; int step; int max_batches; float *scales; int *steps; int num_steps; int burn_in; int adam; float B1; float B2; float eps; int inputs; int outputs; int truths; int notruth; int h, w, c; int max_crop; int min_crop; float max_ratio; float min_ratio; int center; float angle; float aspect; float exposure; float saturation; float hue; int random; int gpu_index; tree *hierarchy; float *input; float *truth; float *delta; float *workspace; int train; int index; float *cost; float clip; } network; typedef struct { int w; int h; int c; float *data; } image; network *load_network(char *cfg, char *weights, int clear); image letterbox_image(image im, int w, int h); int resize_network(network *net, int w, int h); void top_predictions(network *net, int n, int *index); void free_image(image m); image load_image_color(char *filename, int w, int h); float *network_predict_image(network *net, image im); float *network_predict(network *net, float *input); network *make_network(int n); layer make_convolutional_layer(int batch, int h, int w, int c, int n, int groups, int size, int stride, int padding, ACTIVATION activation, int batch_normalize, int binary, int xnor, int adam); layer make_connected_layer(int batch, int inputs, int outputs, ACTIVATION activation, int batch_normalize, int adam); layer make_maxpool_layer(int batch, int h, int w, int c, int size, int stride, int padding); layer make_avgpool_layer(int batch, int w, int h, int c); layer make_shortcut_layer(int batch, int index, int w, int h, int c, int w2, int h2, int c2); layer make_batchnorm_layer(int batch, int w, int h, int c); layer make_reorg_layer(int batch, int w, int h, int c, int stride, int reverse, int flatten, int extra); layer make_region_layer(int batch, int w, int h, int n, int classes, int coords); layer make_softmax_layer(int batch, int inputs, int groups); layer make_rnn_layer(int batch, int inputs, int outputs, int steps, ACTIVATION activation, int batch_normalize, int adam); layer make_yolo_layer(int batch, int w, int h, int n, int total, int *mask, int classes); layer make_crnn_layer(int batch, int h, int w, int c, int hidden_filters, int output_filters, int steps, ACTIVATION activation, int batch_normalize); layer make_lstm_layer(int batch, int inputs, int outputs, int steps, int batch_normalize, int adam); layer make_gru_layer(int batch, int inputs, int outputs, int steps, int batch_normalize, int adam); layer make_upsample_layer(int batch, int w, int h, int c, int stride); layer make_l2norm_layer(int batch, int inputs); void free_network(network *net); """)
ffibuilder.cdef(""" int Pa_GetVersion( void ); const char* Pa_GetVersionText( void ); typedef int PaError; typedef enum PaErrorCode { paNoError = 0, paNotInitialized = -10000, paUnanticipatedHostError, paInvalidChannelCount, paInvalidSampleRate, paInvalidDevice, paInvalidFlag, paSampleFormatNotSupported, paBadIODeviceCombination, paInsufficientMemory, paBufferTooBig, paBufferTooSmall, paNullCallback, paBadStreamPtr, paTimedOut, paInternalError, paDeviceUnavailable, paIncompatibleHostApiSpecificStreamInfo, paStreamIsStopped, paStreamIsNotStopped, paInputOverflowed, paOutputUnderflowed, paHostApiNotFound, paInvalidHostApi, paCanNotReadFromACallbackStream, paCanNotWriteToACallbackStream, paCanNotReadFromAnOutputOnlyStream, paCanNotWriteToAnInputOnlyStream, paIncompatibleStreamHostApi, paBadBufferPtr } PaErrorCode; const char *Pa_GetErrorText( PaError errorCode ); PaError Pa_Initialize( void ); PaError Pa_Terminate( void ); typedef int PaDeviceIndex; #define paNoDevice -1 #define paUseHostApiSpecificDeviceSpecification -2 typedef int PaHostApiIndex; PaHostApiIndex Pa_GetHostApiCount( void ); PaHostApiIndex Pa_GetDefaultHostApi( void ); typedef enum PaHostApiTypeId { paInDevelopment=0, paDirectSound=1, paMME=2, paASIO=3, paSoundManager=4, paCoreAudio=5, paOSS=7, paALSA=8, paAL=9, paBeOS=10, paWDMKS=11, paJACK=12, paWASAPI=13, paAudioScienceHPI=14 } PaHostApiTypeId; typedef struct PaHostApiInfo { int structVersion; PaHostApiTypeId type; const char *name; int deviceCount; PaDeviceIndex defaultInputDevice; PaDeviceIndex defaultOutputDevice; } PaHostApiInfo; const PaHostApiInfo * Pa_GetHostApiInfo( PaHostApiIndex hostApi ); PaHostApiIndex Pa_HostApiTypeIdToHostApiIndex( PaHostApiTypeId type ); PaDeviceIndex Pa_HostApiDeviceIndexToDeviceIndex( PaHostApiIndex hostApi, int hostApiDeviceIndex ); typedef struct PaHostErrorInfo{ PaHostApiTypeId hostApiType; long errorCode; const char *errorText; }PaHostErrorInfo; const PaHostErrorInfo* Pa_GetLastHostErrorInfo( void ); PaDeviceIndex Pa_GetDeviceCount( void ); PaDeviceIndex Pa_GetDefaultInputDevice( void ); PaDeviceIndex Pa_GetDefaultOutputDevice( void ); typedef double PaTime; typedef unsigned long PaSampleFormat; #define paFloat32 0x00000001 #define paInt32 0x00000002 #define paInt24 0x00000004 #define paInt16 0x00000008 #define paInt8 0x00000010 #define paUInt8 0x00000020 #define paCustomFormat 0x00010000 #define paNonInterleaved 0x80000000 typedef struct PaDeviceInfo { int structVersion; const char *name; PaHostApiIndex hostApi; int maxInputChannels; int maxOutputChannels; PaTime defaultLowInputLatency; PaTime defaultLowOutputLatency; PaTime defaultHighInputLatency; PaTime defaultHighOutputLatency; double defaultSampleRate; } PaDeviceInfo; const PaDeviceInfo* Pa_GetDeviceInfo( PaDeviceIndex device ); typedef struct PaStreamParameters { PaDeviceIndex device; int channelCount; PaSampleFormat sampleFormat; PaTime suggestedLatency; void *hostApiSpecificStreamInfo; } PaStreamParameters; #define paFormatIsSupported 0 PaError Pa_IsFormatSupported( const PaStreamParameters *inputParameters, const PaStreamParameters *outputParameters, double sampleRate ); typedef void PaStream; #define paFramesPerBufferUnspecified 0 typedef unsigned long PaStreamFlags; #define paNoFlag 0 #define paClipOff 0x00000001 #define paDitherOff 0x00000002 #define paNeverDropInput 0x00000004 #define paPrimeOutputBuffersUsingStreamCallback 0x00000008 #define paPlatformSpecificFlags 0xFFFF0000 typedef struct PaStreamCallbackTimeInfo{ PaTime inputBufferAdcTime; PaTime currentTime; PaTime outputBufferDacTime; } PaStreamCallbackTimeInfo; typedef unsigned long PaStreamCallbackFlags; #define paInputUnderflow 0x00000001 #define paInputOverflow 0x00000002 #define paOutputUnderflow 0x00000004 #define paOutputOverflow 0x00000008 #define paPrimingOutput 0x00000010 typedef enum PaStreamCallbackResult { paContinue=0, paComplete=1, paAbort=2 } PaStreamCallbackResult; typedef int PaStreamCallback( const void *input, void *output, unsigned long frameCount, const PaStreamCallbackTimeInfo* timeInfo, PaStreamCallbackFlags statusFlags, void *userData ); PaError Pa_OpenStream( PaStream** stream, const PaStreamParameters *inputParameters, const PaStreamParameters *outputParameters, double sampleRate, unsigned long framesPerBuffer, PaStreamFlags streamFlags, PaStreamCallback *streamCallback, void *userData ); PaError Pa_OpenDefaultStream( PaStream** stream, int numInputChannels, int numOutputChannels, PaSampleFormat sampleFormat, double sampleRate, unsigned long framesPerBuffer, PaStreamCallback *streamCallback, void *userData ); PaError Pa_CloseStream( PaStream *stream ); typedef void PaStreamFinishedCallback( void *userData ); PaError Pa_SetStreamFinishedCallback( PaStream *stream, PaStreamFinishedCallback* streamFinishedCallback ); PaError Pa_StartStream( PaStream *stream ); PaError Pa_StopStream( PaStream *stream ); PaError Pa_AbortStream( PaStream *stream ); PaError Pa_IsStreamStopped( PaStream *stream ); PaError Pa_IsStreamActive( PaStream *stream ); typedef struct PaStreamInfo { int structVersion; PaTime inputLatency; PaTime outputLatency; double sampleRate; } PaStreamInfo; const PaStreamInfo* Pa_GetStreamInfo( PaStream *stream ); PaTime Pa_GetStreamTime( PaStream *stream ); double Pa_GetStreamCpuLoad( PaStream* stream ); PaError Pa_ReadStream( PaStream* stream, void *buffer, unsigned long frames ); PaError Pa_WriteStream( PaStream* stream, const void *buffer, unsigned long frames ); signed long Pa_GetStreamReadAvailable( PaStream* stream ); signed long Pa_GetStreamWriteAvailable( PaStream* stream ); PaHostApiTypeId Pa_GetStreamHostApiType( PaStream* stream ); PaError Pa_GetSampleSize( PaSampleFormat format ); void Pa_Sleep( long msec ); /* pa_mac_core.h */ typedef int32_t SInt32; typedef struct { unsigned long size; PaHostApiTypeId hostApiType; unsigned long version; unsigned long flags; SInt32 const * channelMap; unsigned long channelMapSize; } PaMacCoreStreamInfo; void PaMacCore_SetupStreamInfo( PaMacCoreStreamInfo *data, unsigned long flags ); void PaMacCore_SetupChannelMap( PaMacCoreStreamInfo *data, const SInt32 * const channelMap, unsigned long channelMapSize ); const char *PaMacCore_GetChannelName( int device, int channelIndex, bool input ); #define paMacCoreChangeDeviceParameters 0x01 #define paMacCoreFailIfConversionRequired 0x02 #define paMacCoreConversionQualityMin 0x0100 #define paMacCoreConversionQualityMedium 0x0200 #define paMacCoreConversionQualityLow 0x0300 #define paMacCoreConversionQualityHigh 0x0400 #define paMacCoreConversionQualityMax 0x0000 #define paMacCorePlayNice 0x00 #define paMacCorePro 0x01 #define paMacCoreMinimizeCPUButPlayNice 0x0100 #define paMacCoreMinimizeCPU 0x0101 /* pa_win_waveformat.h */ typedef unsigned long PaWinWaveFormatChannelMask; /* pa_asio.h */ #define paAsioUseChannelSelectors 0x01 typedef struct PaAsioStreamInfo { unsigned long size; PaHostApiTypeId hostApiType; unsigned long version; unsigned long flags; int *channelSelectors; } PaAsioStreamInfo; /* pa_win_wasapi.h */ typedef enum PaWasapiFlags { paWinWasapiExclusive = 1, paWinWasapiRedirectHostProcessor = 2, paWinWasapiUseChannelMask = 4, paWinWasapiPolling = 8, paWinWasapiThreadPriority = 16 } PaWasapiFlags; typedef void (*PaWasapiHostProcessorCallback) ( void *inputBuffer, long inputFrames, void *outputBuffer, long outputFrames, void *userData); typedef enum PaWasapiThreadPriority { eThreadPriorityNone = 0, eThreadPriorityAudio, eThreadPriorityCapture, eThreadPriorityDistribution, eThreadPriorityGames, eThreadPriorityPlayback, eThreadPriorityProAudio, eThreadPriorityWindowManager } PaWasapiThreadPriority; typedef enum PaWasapiStreamCategory { eAudioCategoryOther = 0, eAudioCategoryCommunications = 3, eAudioCategoryAlerts = 4, eAudioCategorySoundEffects = 5, eAudioCategoryGameEffects = 6, eAudioCategoryGameMedia = 7, eAudioCategoryGameChat = 8, eAudioCategorySpeech = 9, eAudioCategoryMovie = 10, eAudioCategoryMedia = 11 } PaWasapiStreamCategory; typedef enum PaWasapiStreamOption { eStreamOptionNone = 0, eStreamOptionRaw = 1, eStreamOptionMatchFormat = 2 } PaWasapiStreamOption; typedef struct PaWasapiStreamInfo { unsigned long size; PaHostApiTypeId hostApiType; unsigned long version; unsigned long flags; PaWinWaveFormatChannelMask channelMask; PaWasapiHostProcessorCallback hostProcessorOutput; PaWasapiHostProcessorCallback hostProcessorInput; PaWasapiThreadPriority threadPriority; PaWasapiStreamCategory streamCategory; PaWasapiStreamOption streamOption; } PaWasapiStreamInfo; """)
ffi.cdef(''' struct ExpressionClassHandler; typedef struct ExpressionClassHandler* Expression; struct calculate_c_library_template { void (*version)(char*); void (*author)(char*); void (*date)(char*); void (*constants)(char*); void (*operators)(char*); void (*functions)(char*); Expression (*create)(const char*, const char*, char*); Expression (*build)(const char*, const char*); Expression (*parse)(const char*, char*); void (*free)(Expression); void (*expression)(Expression, char*); void (*variables)(Expression, char*); void (*infix)(Expression, char*); void (*postfix)(Expression, char*); void (*tree)(Expression, char*); double (*evaluate)(Expression, double*, int, char*); double (*eval)(Expression, double*, int); double (*value)(Expression, ...); }; const struct calculate_c_library_template* get_calculate_reference(); ''')
ffi.cdef(""" typedef struct { const char* name; double value; } SignalPackValue; typedef struct { uint32_t address; const char* name; double default_value; } SignalParseOptions; typedef struct { uint32_t address; int check_frequency; } MessageParseOptions; typedef struct { uint32_t address; uint16_t ts; const char* name; double value; } SignalValue; typedef enum { DEFAULT, HONDA_CHECKSUM, HONDA_COUNTER, TOYOTA_CHECKSUM, PEDAL_CHECKSUM, PEDAL_COUNTER, VOLKSWAGEN_CHECKSUM, VOLKSWAGEN_COUNTER, } SignalType; typedef struct { const char* name; int b1, b2, bo; bool is_signed; double factor, offset; SignalType type; } Signal; typedef struct { const char* name; uint32_t address; unsigned int size; size_t num_sigs; const Signal *sigs; } Msg; typedef struct { const char* name; uint32_t address; const char* def_val; const Signal *sigs; } Val; typedef struct { const char* name; size_t num_msgs; const Msg *msgs; const Val *vals; size_t num_vals; } DBC; void* can_init(int bus, const char* dbc_name, size_t num_message_options, const MessageParseOptions* message_options, size_t num_signal_options, const SignalParseOptions* signal_options, bool sendcan, const char* tcp_addr, int timeout); int can_update(void* can, uint64_t sec, bool wait); size_t can_query_latest(void* can, bool *out_can_valid, size_t out_values_size, SignalValue* out_values); const DBC* dbc_lookup(const char* dbc_name); void* canpack_init(const char* dbc_name); uint64_t canpack_pack(void* inst, uint32_t address, size_t num_vals, const SignalPackValue *vals, int counter); """)
def test_cffi_assembly(): mesh = create_unit_square(MPI.COMM_WORLD, 13, 13) V = FunctionSpace(mesh, ("Lagrange", 1)) if mesh.comm.rank == 0: from cffi import FFI ffibuilder = FFI() ffibuilder.set_source( "_cffi_kernelA", r""" #include <math.h> #include <stdalign.h> void tabulate_tensor_poissonA(double* restrict A, const double* w, const double* c, const double* restrict coordinate_dofs, const int* entity_local_index, const int* cell_orientation) { // Precomputed values of basis functions and precomputations // FE* dimensions: [entities][points][dofs] // PI* dimensions: [entities][dofs][dofs] or [entities][dofs] // PM* dimensions: [entities][dofs][dofs] alignas(32) static const double FE3_C0_D01_Q1[1][1][2] = { { { -1.0, 1.0 } } }; // Unstructured piecewise computations const double J_c0 = coordinate_dofs[0] * FE3_C0_D01_Q1[0][0][0] + coordinate_dofs[3] * FE3_C0_D01_Q1[0][0][1]; const double J_c3 = coordinate_dofs[1] * FE3_C0_D01_Q1[0][0][0] + coordinate_dofs[7] * FE3_C0_D01_Q1[0][0][1]; const double J_c1 = coordinate_dofs[0] * FE3_C0_D01_Q1[0][0][0] + coordinate_dofs[6] * FE3_C0_D01_Q1[0][0][1]; const double J_c2 = coordinate_dofs[1] * FE3_C0_D01_Q1[0][0][0] + coordinate_dofs[4] * FE3_C0_D01_Q1[0][0][1]; alignas(32) double sp[20]; sp[0] = J_c0 * J_c3; sp[1] = J_c1 * J_c2; sp[2] = sp[0] + -1 * sp[1]; sp[3] = J_c0 / sp[2]; sp[4] = -1 * J_c1 / sp[2]; sp[5] = sp[3] * sp[3]; sp[6] = sp[3] * sp[4]; sp[7] = sp[4] * sp[4]; sp[8] = J_c3 / sp[2]; sp[9] = -1 * J_c2 / sp[2]; sp[10] = sp[9] * sp[9]; sp[11] = sp[8] * sp[9]; sp[12] = sp[8] * sp[8]; sp[13] = sp[5] + sp[10]; sp[14] = sp[6] + sp[11]; sp[15] = sp[12] + sp[7]; sp[16] = fabs(sp[2]); sp[17] = sp[13] * sp[16]; sp[18] = sp[14] * sp[16]; sp[19] = sp[15] * sp[16]; // UFLACS block mode: preintegrated A[0] = 0.5 * sp[19] + 0.5 * sp[18] + 0.5 * sp[18] + 0.5 * sp[17]; A[1] = -0.5 * sp[19] + -0.5 * sp[18]; A[2] = -0.5 * sp[18] + -0.5 * sp[17]; A[3] = -0.5 * sp[19] + -0.5 * sp[18]; A[4] = 0.5 * sp[19]; A[5] = 0.5 * sp[18]; A[6] = -0.5 * sp[18] + -0.5 * sp[17]; A[7] = 0.5 * sp[18]; A[8] = 0.5 * sp[17]; } void tabulate_tensor_poissonL(double* restrict A, const double* w, const double* c, const double* restrict coordinate_dofs, const int* entity_local_index, const int* cell_orientation) { // Precomputed values of basis functions and precomputations // FE* dimensions: [entities][points][dofs] // PI* dimensions: [entities][dofs][dofs] or [entities][dofs] // PM* dimensions: [entities][dofs][dofs] alignas(32) static const double FE4_C0_D01_Q1[1][1][2] = { { { -1.0, 1.0 } } }; // Unstructured piecewise computations const double J_c0 = coordinate_dofs[0] * FE4_C0_D01_Q1[0][0][0] + coordinate_dofs[3] * FE4_C0_D01_Q1[0][0][1]; const double J_c3 = coordinate_dofs[1] * FE4_C0_D01_Q1[0][0][0] + coordinate_dofs[7] * FE4_C0_D01_Q1[0][0][1]; const double J_c1 = coordinate_dofs[0] * FE4_C0_D01_Q1[0][0][0] + coordinate_dofs[6] * FE4_C0_D01_Q1[0][0][1]; const double J_c2 = coordinate_dofs[1] * FE4_C0_D01_Q1[0][0][0] + coordinate_dofs[4] * FE4_C0_D01_Q1[0][0][1]; alignas(32) double sp[4]; sp[0] = J_c0 * J_c3; sp[1] = J_c1 * J_c2; sp[2] = sp[0] + -1 * sp[1]; sp[3] = fabs(sp[2]); // UFLACS block mode: preintegrated A[0] = 0.1666666666666667 * sp[3]; A[1] = 0.1666666666666667 * sp[3]; A[2] = 0.1666666666666667 * sp[3]; } """) ffibuilder.cdef(""" void tabulate_tensor_poissonA(double* restrict A, const double* w, const double* c, const double* restrict coordinate_dofs, const int* entity_local_index, const int* cell_orientation); void tabulate_tensor_poissonL(double* restrict A, const double* w, const double* c, const double* restrict coordinate_dofs, const int* entity_local_index, const int* cell_orientation); """) ffibuilder.compile(verbose=True) mesh.comm.Barrier() from _cffi_kernelA import ffi, lib ptrA = ffi.cast("intptr_t", ffi.addressof(lib, "tabulate_tensor_poissonA")) integrals = {IntegralType.cell: ([(-1, ptrA)], None)} a = _cpp.fem.Form_float64([V._cpp_object, V._cpp_object], integrals, [], [], False) ptrL = ffi.cast("intptr_t", ffi.addressof(lib, "tabulate_tensor_poissonL")) integrals = {IntegralType.cell: ([(-1, ptrL)], None)} L = _cpp.fem.Form_float64([V._cpp_object], integrals, [], [], False) A = fem.assemble_matrix(a) A.finalize() assert np.isclose(np.sqrt(A.norm_squared()), 56.124860801609124) b = fem.assemble_vector(L) b.scatter_reverse(_cpp.common.ScatterMode.add) assert np.isclose(b.norm(), 0.0739710713711999)
#!/usr/bin/python3 from cffi import FFI source = open("libsec.h", "r").read() header = """ #include <secp256k1.h> #include <secp256k1_extrakeys.h> #include <secp256k1_schnorrsig.h> """ ffi = FFI() ffi.cdef(source) ffi.set_source("_libsec", header, libraries=["secp256k1"]) ffi.compile(verbose=True)
from common.basedir import BASEDIR from common.params import Params from selfdrive.swaglog import cloudlog STAGING_ROOT = "/data/safe_staging" OVERLAY_UPPER = os.path.join(STAGING_ROOT, "upper") OVERLAY_METADATA = os.path.join(STAGING_ROOT, "metadata") OVERLAY_MERGED = os.path.join(STAGING_ROOT, "merged") FINALIZED = os.path.join(STAGING_ROOT, "finalized") NICE_LOW_PRIORITY = ["nice", "-n", "19"] # Workaround for lack of os.link in the NEOS/termux python ffi = FFI() ffi.cdef("int link(const char *oldpath, const char *newpath);") libc = ffi.dlopen(None) def link(src, dest): return libc.link(src.encode(), dest.encode()) class WaitTimeHelper: def __init__(self): self.ready_event = threading.Event() self.shutdown = False signal.signal(signal.SIGTERM, self.graceful_shutdown) signal.signal(signal.SIGINT, self.graceful_shutdown) signal.signal(signal.SIGHUP, self.update_now)
ffibuilder.cdef(""" typedef void* appd_bt_handle; typedef void* appd_exitcall_handle; typedef void* appd_frame_handle; struct appd_config; struct appd_context_config; enum appd_config_log_level { APPD_LOG_LEVEL_TRACE, APPD_LOG_LEVEL_DEBUG, APPD_LOG_LEVEL_INFO, APPD_LOG_LEVEL_WARN, APPD_LOG_LEVEL_ERROR, APPD_LOG_LEVEL_FATAL }; struct appd_config* appd_config_init(); void appd_config_set_app_name(struct appd_config* cfg, const char* app); void appd_config_set_tier_name(struct appd_config* cfg, const char* tier); void appd_config_set_node_name(struct appd_config* cfg, const char* node); void appd_config_set_controller_host(struct appd_config* cfg, const char* host); void appd_context_config_set_controller_host(struct appd_context_config* context_cfg, const char* host); void appd_config_set_controller_port(struct appd_config* cfg, const unsigned short port); void appd_context_config_set_controller_port(struct appd_context_config* context_cfg, const unsigned short port); void appd_config_set_controller_account(struct appd_config* cfg, const char* acct); void appd_context_config_set_controller_account(struct appd_context_config* context_cfg, const char* acct); void appd_config_set_controller_access_key(struct appd_config* cfg, const char* key); void appd_context_config_set_controller_access_key(struct appd_context_config* context_cfg, const char* key); void appd_config_set_controller_use_ssl(struct appd_config* cfg, const unsigned int ssl); void appd_context_config_set_controller_use_ssl(struct appd_context_config* context_cfg, unsigned int ssl); void appd_config_set_controller_http_proxy_host(struct appd_config* cfg, const char* host); void appd_context_config_set_controller_http_proxy_host(struct appd_context_config* context_cfg, const char* host); void appd_config_set_controller_http_proxy_port(struct appd_config* cfg, const unsigned short port); void appd_context_config_set_controller_http_proxy_port(struct appd_context_config* context_cfg, const unsigned short port); void appd_config_set_controller_http_proxy_username(struct appd_config* cfg, const char* user); void appd_context_config_set_controller_http_proxy_username(struct appd_context_config* context_cfg, const char* user); void appd_config_set_controller_http_proxy_password(struct appd_config* cfg, const char* pwd); void appd_context_config_set_controller_http_proxy_password(struct appd_context_config* context_cfg, const char* pwd); void appd_config_set_controller_http_proxy_password_file(struct appd_config* cfg, const char* file); void appd_context_config_set_controller_http_proxy_password_file(struct appd_context_config* context_cfg, const char* file); void appd_config_set_controller_certificate_file(struct appd_config* cfg, const char* file); void appd_context_config_set_controller_certificate_file(struct appd_context_config* context_cfg, const char* file); void appd_config_set_controller_certificate_dir(struct appd_config* cfg, const char* dir); void appd_context_config_set_controller_certificate_dir(struct appd_context_config* context_cfg, const char* dir); void appd_config_set_logging_min_level(struct appd_config* cfg, enum appd_config_log_level lvl); void appd_config_set_logging_log_dir(struct appd_config* cfg, const char* dir); void appd_config_set_logging_max_num_files(struct appd_config* cfg, const unsigned int num); void appd_config_set_logging_max_file_size_bytes(struct appd_config* cfg, const unsigned int size); void appd_config_set_init_timeout_ms(struct appd_config* cfg, const int time); void appd_config_set_flush_metrics_on_shutdown(struct appd_config* cfg, int enable); void appd_config_getenv(struct appd_config* cfg, const char* prefix); struct appd_context_config* appd_context_config_init(const char* context); void appd_context_config_set_app_name(struct appd_context_config* context_cfg, const char* app); void appd_context_config_set_tier_name(struct appd_context_config* context_cfg, const char* tier); void appd_context_config_set_node_name(struct appd_context_config* context_cfg, const char* node); int appd_sdk_add_app_context(struct appd_context_config* context_cfg); int appd_sdk_init(const struct appd_config* config); void appd_backend_declare(const char* type, const char* unregistered_name); int appd_backend_set_identifying_property(const char* backend, const char* key, const char* value); int appd_backend_prevent_agent_resolution(const char* backend); int appd_backend_add(const char* backend); appd_bt_handle appd_bt_begin(const char* name, const char* correlation_header); appd_bt_handle appd_bt_begin_with_app_context(const char* context, const char* name, const char* correlation_header); void appd_bt_store(appd_bt_handle bt, const char* guid); appd_bt_handle appd_bt_get(const char* guid); enum appd_error_level { APPD_LEVEL_NOTICE, APPD_LEVEL_WARNING, APPD_LEVEL_ERROR }; void appd_bt_add_error(appd_bt_handle bt, enum appd_error_level level, const char* message, int mark_bt_as_error); char appd_bt_is_snapshotting(appd_bt_handle bt); void appd_bt_add_user_data(appd_bt_handle bt, const char* key, const char* value); void appd_bt_set_url(appd_bt_handle bt, const char* url); void appd_bt_end(appd_bt_handle bt); appd_exitcall_handle appd_exitcall_begin(appd_bt_handle bt, const char* backend); void appd_exitcall_store(appd_exitcall_handle exitcall, const char* guid); appd_exitcall_handle appd_exitcall_get(const char* guid); int appd_exitcall_set_details(appd_exitcall_handle exitcall, const char* details); const char* appd_exitcall_get_correlation_header(appd_exitcall_handle exitcall); void appd_exitcall_add_error(appd_exitcall_handle exitcall, enum appd_error_level level, const char* message, int mark_bt_as_error); void appd_exitcall_end(appd_exitcall_handle exitcall); enum appd_time_rollup_type { APPD_TIMEROLLUP_TYPE_AVERAGE = 1, APPD_TIMEROLLUP_TYPE_SUM, APPD_TIMEROLLUP_TYPE_CURRENT }; enum appd_cluster_rollup_type { APPD_CLUSTERROLLUP_TYPE_INDIVIDUAL = 1, APPD_CLUSTERROLLUP_TYPE_COLLECTIVE }; enum appd_hole_handling_type { APPD_HOLEHANDLING_TYPE_RATE_COUNTER = 1, APPD_HOLEHANDLING_TYPE_REGULAR_COUNTER }; void appd_custom_metric_add(const char* application_context, const char* metric_path, enum appd_time_rollup_type time_rollup_type, enum appd_cluster_rollup_type cluster_rollup_type, enum appd_hole_handling_type hole_handling_type); void appd_custom_metric_report(const char* application_context, const char* metric_path, long value); enum appd_frame_type { APPD_FRAME_TYPE_CPP = 1 }; appd_frame_handle appd_frame_begin(appd_bt_handle bt, enum appd_frame_type frame_type, const char* class_name, const char* method_name, const char* file, unsigned int line_number); void appd_frame_end(appd_bt_handle bt, appd_frame_handle frame); void appd_sdk_term(); """)
class SimEnv(gym.Env): metadata = {'render.modes': ['human']} def __init__(self): super().__init__() self.init = True def step(self, action): # これを呼ぶときには,actionは決定されている # self.state = self._get_state() # 1stepにつき冒頭と末尾に状態取得? # print(action) # print(action.shape) # sys.exit() # if len(self.agents) == 1: # エージェント数が1だとスカラーになってエラー->暫定対処 # action = action.reshape(-1,1) # 他の避難所の行動を決定するコードを書く dict_actions = {} # key:ノードID, value:ノードID self.tmp_action_matrix = np.zeros( (len(self.agents), len(self.actions)), dtype=float) for i, node_id in enumerate(self.agents): # エージェントの行動 # print("ここでtmp_action_matrixに代入する") # print("action.shape", action.shape) dict_actions[node_id] = self._get_action(action[i]) self.tmp_action_matrix[i, action[i]] = 1. for shelter_id, node_id in enumerate(self.actions): # エージェントではない避難所の行動 if node_id in self.agents: # self.sidの代入と更新タイミングに注意 continue _action = self.others[shelter_id].get_action(self.goal_state) dict_actions[node_id] = self._get_action(_action) # # for other in self.others: # # if shelter_id == self.sid: # self.sidの代入と更新タイミングに注意 # if node_id == self.sid: # self.sidの代入と更新タイミングに注意 # # 自エージェントのactionだけ下で上書き # dict_actions[node_id] = self._get_action(action) # else: # _action = self.others[shelter_id].get_action(self.goal_state) # dict_actions[node_id] = self._get_action(_action) # _action = self._get_action(action) # self.call_traffic_regulation(_action, self.num_step) # print("dict_actions",dict_actions) self.call_traffic_regulation(dict_actions, self.num_step) self.call_iterate(self.cur_time + self.interval) # iterate self.cur_time += self.interval self.update_navi_state() # self.navi_stateを更新するだけ self.state = self._get_state() # self.state = self._get_observation(self.cur_time + self.interval) # iterate # observation = self.state2obsv( self.state, self.id ) # observation = self.state # reward = self._get_reward_time() # reward = self._get_reward() # reward = self._get_reward(self.edge_state) sum_pop = np.sum( self.edge_state) * self.interval / self.num_agents # 累積すると平均移動時間 # if self.flg_reward == "goal": # reward, G = self._get_reward_goal() # self.prev_reward = copy.deepcopy(G) # elif self.flg_reward == "goal_cum": # reward, _ = self._get_reward_goal_cum() # # self.prev_reward = copy.deepcopy(G) # elif self.flg_reward == "edge_wo_T0": # reward = self._get_reward_wo_T0(self.state) # elif self.flg_reward == "speed": # reward = self._get_reward_speed(self.state) # elif self.flg_reward == "speed_w_V0": # reward = self._get_reward_speed_w_V0(self.state) if DEBUG: print("in step(), flg_reward = ", self.flg_reward) if self.flg_reward == "time": reward = self._get_reward_time() elif self.flg_reward == "time_once": reward = self._get_reward_total_time_once() # elif self.flg_reward == "total_time_once_wo_norm": # reward = self._get_reward_total_time_once_wo_norm() else: # self.flg_reward == "edge": reward = self._get_reward() # self.episode_reward += sum_pop self.episode_reward += reward # print("CURRENT", self.cur_time, action, sum_pop, self.T_open[self.num_step], reward, self.episode_reward) if DEBUG: print("CURRENT", self.env_id, self.cur_time, action, sum_pop, reward, self.episode_reward) with open(self.resdir + "/current_log.txt", "a") as f: f.write("CURRENT {:} {:} {:} {:} {:} {:}\n".format( self.env_id, self.cur_time, action, sum_pop, reward, self.episode_reward)) self.num_step += 1 done = self.max_step <= self.num_step print(self.max_step, self.num_step, done) # done = False # travel_time = self.mk_travel_open() info = {} if done: agentid, travel_time = self._goal_time_all() # 歩行者の移動速度リストを取得 info = { "episode": { "r": self.episode_reward }, "events": self.event_history, "env_id": self.env_id, "travel_time": travel_time, "agentid": agentid } # print("info",info) if DEBUG: print(self.state.shape, reward, done) # if DEBUG: print(self.state.shape, reward, done, info) return self.state, reward, done, info # obseration, reward, done, info def reset(self): # config = configparser.ConfigParser() # config.read('config.ini') self.sim_time = self.config.getint('SIMULATION', 'sim_time') self.interval = self.config.getint('SIMULATION', 'interval') self.max_step = int(np.ceil(self.sim_time / self.interval)) self.cur_time = 0 self.num_step = 0 self.state = np.zeros(self.num_obsv * self.obs_step) self.episode_reward = 0 self.event_history = [] # self.flag = True # for reward selection # self.prev_goal = 0 self.reset_sim() # set up simulation return self.state def render(self, mode='human', close=False): pass def close(self): pass def seed(self, seed=None, env_id=None, datadirs=None, config=None, R_base=(None, None)): print(R_base) self.T_open, self.travel_open = R_base # print("T_open @ seed",self.T_open) # print("travel_open @ seed",self.travel_open) # training_targets = dict_target["training"] # fixed_agents = dict_target["fixed"] # その他を固定しよう # rule_agents = dict_target["rule"] # fixed_agents: モデルで行動,更新なし,の避難所 # training_targets: 学習対象の避難所 # rule_agents: ルールベースの避難所 # from init (for config import) self.config = config # num_parallel = config.getint('TRAINING', 'num_parallel') # tmp_id = len(training_targets) % num_parallel # tmp_id = seed % len(training_targets) tmp_id = env_id % len(datadirs) # if DEBUG: print(training_targets, tmp_id) self.env_id = env_id # self.sid = training_targets[tmp_id] # self.training_target = self.sid # 不要かも self.datadir = datadirs[tmp_id] # config = configparser.ConfigParser() # config.read('config.ini') # self.num_agents = config.getint('SIMULATION', 'num_agents') # self.num_edges = config.getint('SIMULATION', 'num_edges') self.obs_step = config.getint('TRAINING', 'obs_step') self.obs_degree = config.getint('TRAINING', 'obs_degree') # self.datadir = config.get('SIMULATION', 'datadir') self.tmp_resdir = config['TRAINING']['resdir'] self.actions = np.loadtxt(config['SIMULATION']['actionfn'], dtype=int) # self.agents = training_targets # = self.actions self.agents = copy.deepcopy(self.actions) if DEBUG: print(self.actions) # sys.exit() # self.dict_action = {} # for action in list( self.actions ): # self.dict_action[] self.flg_reward = config['TRAINING']['flg_reward'] # self.flag = True # self.edges = Edge(self.obs_degree) # degreeは不要になったはず... self.edges = Edge(self.datadir) # degreeは不要になったはず... # ->seed()の前に設定してしまいたい self.num_edges = self.edges.num_obsv_edge self.num_goals = self.edges.num_obsv_goal self.num_navi = len(self.actions) * len( self.actions) # 誘導の状態数は,ワンホットベクトルを想定 self.navi_state = np.zeros(len(self.actions) * len(self.actions), dtype=float) # 入れ物だけ作っておく # self.num_navi = len(self.actions) * len(self.agents) # 誘導の状態数は,ワンホットベクトルを想定 # self.navi_state = np.zeros(len(self.actions) * len(self.agents), dtype=float) # 入れ物だけ作っておく # self.num_obsv = self.num_edges + self.num_goals # 1ステップ分の観測の数 if DEBUG: print("self.navi_state.shape", self.navi_state.shape) self.num_obsv = self.num_edges + self.num_goals + self.num_navi # 1ステップ分の観測の数 self.action_space = gym.spaces.Discrete(self.actions.shape[0]) self.observation_space = gym.spaces.Box( low=0, high=100000, # high=self.num_agents, shape=np.zeros(self.num_obsv * self.obs_step).shape) assert self.action_space.n == self.actions.shape[0] assert self.observation_space.shape[0] == self.num_obsv * self.obs_step # self.state = None # self.state = np.zeros(self.num_edges * self.obs_step) # self.cur_time = 0 # self.interval # self.prev_goal = 0 # self.reset() # copy from reset() self.sim_time = self.config.getint('SIMULATION', 'sim_time') self.interval = self.config.getint('SIMULATION', 'interval') self.max_step = int(np.ceil(self.sim_time / self.interval)) self.cur_time = 0 self.num_step = 0 self.state = np.zeros(self.num_obsv * self.obs_step) # original seed # self.np_random, seed = seeding.np_random(seed) # https://harald.co/2019/07/30/reproducibility-issues-using-openai-gym/ seeding.np_random(seed) self.set_datadir(self.datadir) # print(self.datadir) self.set_resdir("%s/sim_result_%d" % (self.tmp_resdir, self.env_id)) # ルールベースの避難所のエージェントを生成する # self.others = {} # for shelter_id, node_id in enumerate( self.actions ): # # 自分のエージェントを作ってもいいけど,使わない # controler = FixControler(shelter_id, self.edges.DistanceMatrix) # self.others[shelter_id] = controler return [seed] def _get_action(self, action): return self.actions[action] # def _get_reward_goal(self): # sum of people who reached goal # G = np.sum( self._goal_cnt() ) # G_diff = G - self.prev_goal # reward = G_diff / self.num_agents # - (1./ self.max_step) # return reward, G # def _get_reward_goal_cum(self): # sum of people who reached goal # G = np.sum( self._goal_cnt() ) # reward = G / self.num_agents # - (1./ self.max_step) # return reward, G def _get_reward_time(self): # mean travel time of people who reached goal agentid, travel_time = self._goal_time() # print(agentid, travel_time) if len(agentid) == 0: return 0 reward = np.sum(self.travel_open[agentid] - travel_time) / np.sum( self.travel_open[agentid]) if reward < 0: return max(reward, -1) return min(reward, 1) def _get_reward_total_time_once( self): # mean travel time of people who reached goal if self.travel_open is None: return 0 if self.max_step > self.num_step + 1: return 0 # reward only last step agentid, travel_time = self._goal_time_all() # print(agentid, travel_time) # if len(agentid) == 0: # return 0 reward = np.sum(self.travel_open[agentid] - travel_time) / np.sum( self.travel_open[agentid]) # reward = np.sum( self.T_open[agentid] - travel_time ) / np.sum( self.T_open[agentid] ) if reward < 0: return max(reward, -1) return min(reward, 1) # def _get_reward_total_time_once_wo_norm(self): # mean travel time of people who reached goal # if self.max_step > self.num_step + 1: # return 0 # reward only last step # agentid, travel_time = self._goal_time_all() # # print(agentid, travel_time) # # if len(agentid) == 0: # # return 0 # reward = np.sum( - travel_time ) / ( 1. * self.sim_time * self.num_agents) # if reward < 0: # return max(reward, -1) # return min(reward, 1) # def mk_travel_open(self): # travel time of people who reached goal # stop_time = (self.max_step + 1) * self.interval # start_time = 0 # print(start_time, stop_time) # tmp = self.lib.goalAgentCnt(start_time, stop_time-1, -1) # all goal # res = self.ffi.new("int[%d][3]" % tmp) # l = self.lib.goalAgent(start_time, stop_time-1, tmp+1, res) # # print(l) # travel_time = np.array( [res[i][1] for i in range(l)] ) # # print(travel_time) # return travel_time # def _get_reward_speed(self, observation): # # moving speedをRewardとする # # 最大値を取るか,平均を取るかは要検討 # # 全歩行者数でRewardを決定 # v_mean = self.speed # th = 1.8 / (v_mean + 0.3) # num_agent = observation[self.num_obsv * (self.obs_step-1):] # observationのサイズを変更したので要修正 # rho = num_agent / (self.edges.dist * self.edges.width) # # rho = rho[self.num_edges * (self.obs_step-1):] # rho_ = np.where(rho == 0, float('-inf'), rho) # v_1 = np.where((0 <= rho) & (rho < th), v_mean, 0) # v_2 = np.where((th <= rho_) & (rho_ < 6), 1.8 / rho_ - 0.3, 0) # v = (v_1 + v_2) * num_agent # weighted # # reward = (v_mean - v) / v_mean # if np.sum(num_agent) == 0: # v = v_mean # else: # v = np.sum(v) / np.sum(num_agent) # reward = ( v_mean - v ) / v_mean # return -reward # def _get_reward_speed_w_V0(self, observation): # # moving speedをRewardとする # # 最大値を取るか,平均を取るかは要検討 # # 全歩行者数でRewardを決定 # v_mean = self.speed # th = 1.8 / (v_mean + 0.3) # num_agent = observation[self.num_obsv * (self.obs_step-1):] # rho = num_agent / (self.edges.dist * self.edges.width) # # rho = rho[self.num_edges * (self.obs_step-1):] # rho_ = np.where(rho == 0, float('-inf'), rho) # v_1 = np.where((0 <= rho) & (rho < th), v_mean, 0) # v_2 = np.where((th <= rho_) & (rho_ < 6), 1.8 / rho_ - 0.3, 0) # v = (v_1 + v_2) * num_agent # weighted # # reward = (v_mean - v) / v_mean # if np.sum(num_agent) == 0: # v = v_mean # else: # v = np.sum(v) / np.sum(num_agent) # # reward = ( v_mean - v ) / v_mean # if self.V_open[self.num_step] == 0: # if v == 0: # return 0 # else: # return 1 # reward = (self.V_open[self.num_step] - v) / (self.V_open[self.num_step]) # if reward < 0: # return max(reward, -1) # return min(reward, 1) # def _get_reward_wo_T0(self, observation): # reward = ( self.num_agents - self._get_num_traveler(observation)) / (1. * self.num_agents) # return reward def _get_reward(self): # t_open tmp_state = np.sum(self.edge_state) * self.interval / self.num_agents # print("T_open",self.T_open) if self.T_open is None: return tmp_state else: if self.T_open[self.num_step] == 0: if tmp_state == 0: return 0 else: return -1 reward = (self.T_open[self.num_step] - tmp_state) / (self.T_open[self.num_step]) # print("tmp_reward", reward) if reward < 0: return max(reward, -1) return min(reward, 1) # def _get_travel_time(self, observation): # # observation = observation[self.num_obsv * (self.obs_step-1):] # observation = observation[self.num_edges * (self.obs_step-1):] # return np.sum(observation) * self.interval / self.num_agents # def _get_num_traveler(self, observation): # observation = observation[self.num_obsv * (self.obs_step-1):] # return np.sum(observation) def call_traffic_regulation(self, actions, t): """ Take action, i.e., open/close gates """ # pass events = self.get_events(actions, t) # for nid, action in actions.items(): # events.append(self.get_events(nid, action, t)) # events = self.get_events(action, t) for e in events: self.lib.setBombDirect(e.encode('ascii')) # if DEBUG: print(e) self.event_history.append(e) return events def _edge_cnt(self): ret = np.zeros(len(self.edges.observed_edge)) for e, idx in self.edges.observed_edge.items(): fr, to = e ret[idx] = self.lib.cntOnEdge(fr - 1, to - 1) return ret def _goal_cnt(self): # ゴールした人の累積人数 ret = np.zeros(len(self.edges.observed_goal)) # 返値は,observed_goal次元 stop_time = self.cur_time # + self.interval start_time = 0 # self.cur_time #stop_time - self.interval for node, idx in sorted(self.edges.observed_goal.items()): tmp = self.lib.goalAgentCnt(start_time, stop_time - 1, node - 1) ret[idx] = tmp # print(start_time, stop_time, node-1, idx, tmp) return ret def _goal_time(self): # 直近のstepにゴールした人の移動時間を求める stop_time = self.cur_time # + self.interval start_time = self.cur_time - self.interval tmp = self.lib.goalAgentCnt(start_time, stop_time - 1, -1) # all goal res = self.ffi.new("int[%d][3]" % tmp) l = self.lib.goalAgent(start_time, stop_time - 1, tmp + 1, res) agentid = np.array([res[i][0] for i in range(l)]) travel_time = np.array([res[i][1] for i in range(l)]) return agentid, travel_time def _goal_time_all(self): # 現時点までにゴールした人の移動時間を求める stop_time = self.cur_time # + self.interval start_time = 0 tmp = self.lib.goalAgentCnt(start_time, stop_time - 1, -1) # all goal res = self.ffi.new("int[%d][3]" % tmp) l = self.lib.goalAgent(start_time, stop_time - 1, tmp + 1, res) agentid = np.array([res[i][0] for i in range(l)]) travel_time = np.array([res[i][1] for i in range(l)]) return agentid, travel_time # def call_edge_cnt(self, stop_time=0): # """ # Count the number of agents on the edge # """ # self.lib.setStop(stop_time) # self.lib.iterate() # ret = self._edge_cnt() # return ret # def call_speed(self, stop_time=0): # """ # Measure the mean of agents speed # """ # # self.lib.setStop(stop_time) # # self.lib.iterate() # print("stop_time : ", stop_time) # # self.state = self._get_observation(stop_time) # iterate # self.call_iterate(stop_time) # self.state = self._get_state() # # observation = state2obsv( self.state, self.id ) # observation = self.state # v_mean = self.speed # th = 1.8 / (v_mean + 0.3) # num_agent = observation[self.num_edges * (self.obs_step-1):] # rho = num_agent / (self.edges.dist * self.edges.width) # # rho = rho[self.num_edges * (self.obs_step-1):] # rho_ = np.where(rho == 0, float('-inf'), rho) # v_1 = np.where((0 <= rho) & (rho < th), v_mean, 0) # v_2 = np.where((th <= rho_) & (rho_ < 6), 1.8 / rho_ - 0.3, 0) # v = (v_1 + v_2) * num_agent # weighted # # reward = (v_mean - v) / v_mean # if np.sum(num_agent) == 0: # v = v_mean # else: # v = np.sum(v) / np.sum(num_agent) # return v # def call_goal_cnt(self, stop_time=0): # """ # Count the number of agents on the goal # """ # self.lib.setStop(stop_time) # self.lib.iterate() # # ret = np.zeros(len(self.edges.dict_edge)) # # for e, idx in self.edges.dict_edge.items(): # ret = _goal_cnt() # return ret def call_iterate(self, stop_time=0): self.lib.setStop(stop_time) self.lib.iterate() def update_navi_state(self): # この関数,本当に必要? # self.navi_state = np.zeros(len(self.actions) * len(self.agents)) # 入れ物だけ作っておく # self.navi_state = self.tmp_action_matrix.reshape(1,-1) self.navi_state = self.tmp_action_matrix.flatten() # print( self.navi_state ) # print( self.navi_state.shape ) def _get_state(self): # 時刻は進めない->繰り返し使うと同じstateが最大4step分だけ繰り返されるので注意 # 1ステップ分ずらす obs = self.state[self.num_obsv:] # 左端を捨てる # 避難所の状況を取得 tmp_goal_state = copy.deepcopy(self._goal_cnt()) # print(tmp_goal_state) self.edge_state = copy.deepcopy(self._edge_cnt()) # 何度も使いそうなので保存 self.goal_state = self.edges.goal_capa - tmp_goal_state # 何度も使いそうなので保存 # print(self.goal_state) # print(np.sum(tmp_goal_state), np.sum(self.edge_state)) cur_obs = np.append(self.edge_state, self.goal_state) cur_obs = np.append(cur_obs, self.navi_state) return np.append(obs, cur_obs) # 右端に追加 # def _get_observation(self, stop_time=0): # self.call_iterate(stop_time) # return self._get_state() # # obs = self.state[self.num_obsv:] # # cur_obs = self.call_edge_cnt(stop_time) # iterate # # return np.append(obs, cur_obs) def set_datadir(self, datadir): self.datadir = datadir # agentfn = os.path.dirname(os.path.abspath(__file__)) + "/../%s/agentlist.txt"%self.datadir agentfn = "%s/agentlist.txt" % self.datadir self.speed = self.get_speed(agentfn) self.num_agents = self.get_num_agents(agentfn) def set_resdir(self, resdir): # print(resdir) os.makedirs(resdir, exist_ok=True) self.resdir = resdir def get_events(self, actions, t): # nid: 誘導元のノードID # action: 誘導先のノードID """ たらい回さない: cmd_name="del_signage_table" tmp = [t,cmd_name,shelter, shelter] たらい回す: cmd_name = "signage" ratio = 1.0 via_length = 0 tmp = [t, cmd_name, shelter, shelter, target_node, target_node, ratio, via_length] # 末尾(1と0)は,分配率と経由地数 """ time = t * self.interval + 1 out = [] out.append("%d clear_all_signage" % time) for nid, action in sorted(actions.items()): if action == -1: pass # out = "%d del_signage_table %d %d"%(time, nid, nid) else: # detour = self.dict_action[action] detour = action # if nid == action: # out = "%d del_signage_table %d %d"%(time, nid, nid) if nid != action: out.append("%d signage %d %d %d %d 1.0 0" % (time, nid, nid, detour, detour)) return out # def call_open_event(self): # # print(self.seed) # # config = configparser.ConfigParser() # # config.read('config.ini') # # datadir = "mkUserlist/data/N80000r{:}i0".format(self.file_seed) # datadir = self.datadir # agentfn = os.path.dirname(os.path.abspath(__file__)) + "/../%s/agentlist.txt"%self.datadir # graphfn = os.path.dirname(os.path.abspath(__file__)) + "/../%s/graph.twd"%self.datadir # goalfn = os.path.dirname(os.path.abspath(__file__)) + "/../%s/goallist.txt"%self.datadir # sim_time = self.config.get('SIMULATION', 'sim_time') # resdir = self.resdir # # resdir = "result/%s"%self.datadir # print(resdir) # os.makedirs(resdir, exist_ok=True) # argv = [sys.argv[0]] # argv.extend([ # agentfn, # graphfn, # goalfn, # "-o", # # "bin/result%s"%self.seed, # resdir, # "-l", # "9999999", # # "300", # ログ周期 # # "10", # ログ周期 # "-e", # sim_time, # # "-S" # ]) # # print(argv) # tmp = [] # for a in argv: # # tmp.append(self.ffi.new("char []", a)) # # tmp.append(self.ffi.new("char []", a.encode('UTF-8'))) # tmp.append(self.ffi.new("char []", a.encode('ascii'))) # argv = self.ffi.new("char *[]", tmp) # # call simulator # self.lib.init(len(argv), argv) # # _action = self._get_action(0) # # print(_action) # # self.call_traffic_regulation({}, 0) # # self.T_open = None # # call_open_event全体をコメントアウトすると動作しないので,ここまでだけ実行(暫定) # return # # elif "speed_w_V0" == self.flg_reward: # # print("self.call_speed") # # self.V_open = [self.call_speed((i + 1) * self.interval) for i in range(self.max_step)] # # print(self.V_open, np.sum(self.V_open)) # # elif "time" == self.flg_reward or "time_once" == self.flg_reward: # # print("travel time") # # self.call_iterate( (self.max_step + 1) * self.interval) # # self.travel_open = self.mk_travel_open() # ここにベースラインを保存する # # print(self.travel_open, len(self.travel_open), np.mean(self.travel_open)) # # else: # # pass def str_cdef(self): ret = """ void init(int argc, char** argv); int setStop(int t); void iterate(); int cntDest(int node, double radius); int cntSrc(int node, double radius); void setBomb( char *fn); int cntOnEdge(int fr, int to); void setBombDirect(char *text); void restart(); void save_ulog(char *fn); void init_restart(int argc, char** argv); int goalAgentCnt(int stime, int etime, int cnt); int goalAgent(int stime, int etime,int n, int result[][3]); """ return ret def set_ffi(self): libsimfn = os.path.dirname( os.path.abspath(__file__)) + "/../bin/libsim.so" self.ffi = FFI() self.lib = self.ffi.dlopen(libsimfn) self.ffi.cdef(self.str_cdef()) # self.call_open_event() datadir = self.datadir # agentfn = os.path.dirname(os.path.abspath(__file__)) + "/../%s/agentlist.txt"%self.datadir # graphfn = os.path.dirname(os.path.abspath(__file__)) + "/../%s/graph.twd"%self.datadir # goalfn = os.path.dirname(os.path.abspath(__file__)) + "/../%s/goallist.txt"%self.datadir agentfn = "%s/agentlist.txt" % self.datadir graphfn = "%s/graph.twd" % self.datadir goalfn = "%s/goallist.txt" % self.datadir sim_time = self.config.get('SIMULATION', 'sim_time') resdir = self.resdir # resdir = "result/%s"%self.datadir print(resdir) os.makedirs(resdir, exist_ok=True) argv = [sys.argv[0]] argv.extend([ agentfn, graphfn, goalfn, "-o", # "bin/result%s"%self.seed, resdir, "-l", "9999999", # "300", # ログ周期 # "10", # ログ周期 "-e", sim_time, # "-S" ]) # print(argv) tmp = [] for a in argv: # tmp.append(self.ffi.new("char []", a)) # tmp.append(self.ffi.new("char []", a.encode('UTF-8'))) tmp.append(self.ffi.new("char []", a.encode('ascii'))) argv = self.ffi.new("char *[]", tmp) # call simulator self.lib.init(len(argv), argv) def reset_sim(self): print("reset simulator configure") # if True: if self.init: self.set_ffi() # libsimfn = os.path.dirname(os.path.abspath(__file__)) + "/../bin/libsim.so" # self.ffi = FFI() # self.lib = self.ffi.dlopen(libsimfn) # self.ffi.cdef(self.str_cdef()) # self.call_open_event() # self.num_step = 0 # call_open_event中にstep進めてしまってるので # self.cur_time = 0 # 同上 print("INIT") # save initial state # 以下コメントアウトで,毎回ffi作成->エラーになる self.init = False # else: # # load inital state # self.lib.restart() self.lib.restart() def get_speed(self, agentfn): with open(agentfn) as f: lines = f.readlines() return sum([float(l.split('\t')[2]) for l in lines[1:]]) / float(lines[0].split(' ')[0]) def get_num_agents(self, agentfn): with open(agentfn) as f: lines = f.readlines() return int(lines[0].split(' ')[0])