def test_vararg(self):
if not sys.platform.startswith('linux'):
py.test.skip("probably no symbol 'stderr' in the lib")
ffi = FFI(backend=self.Backend())
ffi.cdef("""
int fprintf(void *, const char *format, ...);
void *stderr;
""")
ffi.C = ffi.dlopen(None)
with FdWriteCapture() as fd:
ffi.C.fprintf(ffi.C.stderr, b"hello with no arguments\n")
ffi.C.fprintf(ffi.C.stderr,
b"hello, %s!\n", ffi.new("char[]", b"world"))
ffi.C.fprintf(ffi.C.stderr,
ffi.new("char[]", b"hello, %s!\n"),
ffi.new("char[]", b"world2"))
ffi.C.fprintf(ffi.C.stderr,
b"hello int %d long %ld long long %lld\n",
ffi.cast("int", 42),
ffi.cast("long", 84),
ffi.cast("long long", 168))
ffi.C.fprintf(ffi.C.stderr, b"hello %p\n", ffi.NULL)
res = fd.getvalue()
assert res == (b"hello with no arguments\n"
b"hello, world!\n"
b"hello, world2!\n"
b"hello int 42 long 84 long long 168\n"
b"hello (nil)\n")
def test_redefine_common_type():
prefix = "" if sys.version_info < (3,) else "b"
ffi = FFI()
ffi.cdef("typedef char FILE;")
assert repr(ffi.cast("FILE", 123)) == "<cdata 'char' %s'{'>" % prefix
ffi.cdef("typedef char int32_t;")
assert repr(ffi.cast("int32_t", 123)) == "<cdata 'char' %s'{'>" % prefix
def test_from_buffer(self):
import array
ffi = FFI()
a = array.array('H', [10000, 20000, 30000])
c = ffi.from_buffer(a)
assert ffi.typeof(c) is ffi.typeof("char[]")
ffi.cast("unsigned short *", c)[1] += 500
assert list(a) == [10000, 20500, 30000]
def test_redefine_common_type():
prefix = "" if sys.version_info < (3,) else "b"
ffi = FFI()
ffi.cdef("typedef char FILE;")
assert repr(ffi.cast("FILE", 123)) == "<cdata 'char' %s'{'>" % prefix
ffi.cdef("typedef char int32_t;")
assert repr(ffi.cast("int32_t", 123)) == "<cdata 'char' %s'{'>" % prefix
ffi = FFI()
ffi.cdef("typedef int bool, *FILE;")
assert repr(ffi.cast("bool", 123)) == "<cdata 'int' 123>"
assert repr(ffi.cast("FILE", 123)) == "<cdata 'int *' 0x7b>"
ffi = FFI()
ffi.cdef("typedef bool (*fn_t)(bool, bool);") # "bool," but within "( )"
def test_function_pointer(self):
ffi = FFI(backend=self.Backend())
def cb(charp):
assert repr(charp).startswith("<cdata 'char *' 0x")
return 42
fptr = ffi.callback("int(*)(const char *txt)", cb)
assert fptr != ffi.callback("int(*)(const char *)", cb)
assert repr(fptr) == "<cdata 'int(*)(char *)' calling %r>" % (cb,)
res = fptr(b"Hello")
assert res == 42
#
if not sys.platform.startswith('linux'):
py.test.skip("probably no symbol 'stderr' in the lib")
ffi.cdef("""
int fputs(const char *, void *);
void *stderr;
""")
ffi.C = ffi.dlopen(None)
fptr = ffi.cast("int(*)(const char *txt, void *)", ffi.C.fputs)
assert fptr == ffi.C.fputs
assert repr(fptr).startswith("<cdata 'int(*)(char *, void *)' 0x")
with FdWriteCapture() as fd:
fptr(b"world\n", ffi.C.stderr)
res = fd.getvalue()
assert res == b'world\n'
def test_win_common_types():
from cffi.commontypes import COMMON_TYPES, _CACHE
from cffi.commontypes import win_common_types, resolve_common_type
#
def clear_all(extra={}, old_dict=COMMON_TYPES.copy()):
COMMON_TYPES.clear()
COMMON_TYPES.update(old_dict)
COMMON_TYPES.update(extra)
_CACHE.clear()
#
for maxsize in [2 ** 32 - 1, 2 ** 64 - 1]:
ct = win_common_types(maxsize)
clear_all(ct)
for key in sorted(ct):
resolve_common_type(key)
# assert did not crash
# now try to use e.g. WPARAM (-> UINT_PTR -> unsigned 32/64-bit)
for maxsize in [2 ** 32 - 1, 2 ** 64 - 1]:
ct = win_common_types(maxsize)
clear_all(ct)
ffi = FFI()
value = int(ffi.cast("WPARAM", -1))
assert value == maxsize
#
clear_all()
def test_some_integer_type():
ffi = FFI()
ffi.cdef("""
typedef int... foo_t;
typedef unsigned long... bar_t;
typedef struct { foo_t a, b; } mystruct_t;
foo_t foobar(bar_t, mystruct_t);
static const bar_t mu = -20;
static const foo_t nu = 20;
""")
lib = verify(ffi, 'test_some_integer_type', """
typedef unsigned long long foo_t;
typedef short bar_t;
typedef struct { foo_t a, b; } mystruct_t;
static foo_t foobar(bar_t x, mystruct_t s) {
return (foo_t)x + s.a + s.b;
}
static const bar_t mu = -20;
static const foo_t nu = 20;
""")
assert ffi.sizeof("foo_t") == ffi.sizeof("unsigned long long")
assert ffi.sizeof("bar_t") == ffi.sizeof("short")
maxulonglong = 2 ** 64 - 1
assert int(ffi.cast("foo_t", -1)) == maxulonglong
assert int(ffi.cast("bar_t", -1)) == -1
assert lib.foobar(-1, [0, 0]) == maxulonglong
assert lib.foobar(2 ** 15 - 1, [0, 0]) == 2 ** 15 - 1
assert lib.foobar(10, [20, 31]) == 61
assert lib.foobar(0, [0, maxulonglong]) == maxulonglong
py.test.raises(OverflowError, lib.foobar, 2 ** 15, [0, 0])
py.test.raises(OverflowError, lib.foobar, -(2 ** 15) - 1, [0, 0])
py.test.raises(OverflowError, ffi.new, "mystruct_t *", [0, -1])
assert lib.mu == -20
assert lib.nu == 20
def test_ffi_new_allocator_2(self):
ffi = FFI(backend=self.Backend())
seen = []
def myalloc(size):
seen.append(size)
return ffi.new("char[]", b"X" * size)
def myfree(raw):
seen.append(raw)
alloc1 = ffi.new_allocator(myalloc, myfree)
alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree,
should_clear_after_alloc=False)
p1 = alloc1("int[10]")
p2 = alloc2("int[]", 10)
assert seen == [40, 40]
assert ffi.typeof(p1) == ffi.typeof("int[10]")
assert ffi.sizeof(p1) == 40
assert ffi.typeof(p2) == ffi.typeof("int[]")
assert ffi.sizeof(p2) == 40
assert p1[5] == 0
assert p2[6] == ord('X') * 0x01010101
raw1 = ffi.cast("char *", p1)
raw2 = ffi.cast("char *", p2)
del p1, p2
retries = 0
while len(seen) != 4:
retries += 1
assert retries <= 5
import gc; gc.collect()
assert seen == [40, 40, raw1, raw2]
assert repr(seen[2]) == "<cdata 'char[]' owning 41 bytes>"
assert repr(seen[3]) == "<cdata 'char[]' owning 41 bytes>"
def test_new_handle(self):
ffi = FFI(backend=self.Backend())
o = [2, 3, 4]
p = ffi.new_handle(o)
assert ffi.typeof(p) == ffi.typeof("void *")
assert ffi.from_handle(p) is o
assert ffi.from_handle(ffi.cast("char *", p)) is o
py.test.raises(RuntimeError, ffi.from_handle, ffi.NULL)
def range_f(start, end, step):
"""Create a list of double values."""
end = c_double(end)
ffi = FFI()
start = c_double(start)
while start.value <= end.value:
yield ffi.cast("double", start.value)
start.value = start.value + c_double(step).value
def check(self, source, expected_ofs_y, expected_align, expected_size):
# NOTE: 'expected_*' is the numbers expected from GCC.
# The numbers expected from MSVC are not explicitly written
# in this file, and will just be taken from the compiler.
ffi = FFI()
ffi.cdef("struct s1 { %s };" % source)
ctype = ffi.typeof("struct s1")
# verify the information with gcc
ffi1 = FFI()
ffi1.cdef(
"""
static const int Gofs_y, Galign, Gsize;
struct s1 *try_with_value(int fieldnum, long long value);
"""
)
fnames = [name for name, cfield in ctype.fields if name and cfield.bitsize > 0]
setters = ["case %d: s.%s = value; break;" % iname for iname in enumerate(fnames)]
lib = ffi1.verify(
"""
struct s1 { %s };
struct sa { char a; struct s1 b; };
#define Gofs_y offsetof(struct s1, y)
#define Galign offsetof(struct sa, b)
#define Gsize sizeof(struct s1)
struct s1 *try_with_value(int fieldnum, long long value)
{
static struct s1 s;
memset(&s, 0, sizeof(s));
switch (fieldnum) { %s }
return &s;
}
"""
% (source, " ".join(setters))
)
if sys.platform == "win32":
expected_ofs_y = lib.Gofs_y
expected_align = lib.Galign
expected_size = lib.Gsize
else:
assert (lib.Gofs_y, lib.Galign, lib.Gsize) == (expected_ofs_y, expected_align, expected_size)
# the real test follows
assert ffi.offsetof("struct s1", "y") == expected_ofs_y
assert ffi.alignof("struct s1") == expected_align
assert ffi.sizeof("struct s1") == expected_size
# compare the actual storage of the two
for name, cfield in ctype.fields:
if cfield.bitsize < 0 or not name:
continue
if int(ffi.cast(cfield.type, -1)) == -1: # signed
min_value = -(1 << (cfield.bitsize - 1))
max_value = (1 << (cfield.bitsize - 1)) - 1
else:
min_value = 0
max_value = (1 << cfield.bitsize) - 1
for t in [1, 2, 4, 8, 16, 128, 2813, 89728, 981729, -1, -2, -4, -8, -16, -128, -2813, -89728, -981729]:
if min_value <= t <= max_value:
self._fieldcheck(ffi, lib, fnames, name, t)
def test_WPARAM_on_windows():
if sys.platform != 'win32':
py.test.skip("Only for Windows")
ffi = FFI()
ffi.cdef("void f(WPARAM);")
#
# WPARAM -> UINT_PTR -> unsigned 32/64-bit integer
ffi = FFI()
value = int(ffi.cast("WPARAM", -42))
assert value == sys.maxsize * 2 - 40
def test_cannot_instantiate_manually(self):
ffi = FFI()
ct = type(ffi.typeof("void *"))
py.test.raises(TypeError, ct)
py.test.raises(TypeError, ct, ffi.NULL)
for cd in [type(ffi.cast("void *", 0)),
type(ffi.new("char[]", 3)),
type(ffi.gc(ffi.NULL, lambda x: None))]:
py.test.raises(TypeError, cd)
py.test.raises(TypeError, cd, ffi.NULL)
py.test.raises(TypeError, cd, ffi.typeof("void *"))
def test_explicit_cdecl_stdcall(self):
if sys.platform != 'win32':
py.test.skip("Windows-only test")
if self.Backend is CTypesBackend:
py.test.skip("not with the ctypes backend")
win64 = (sys.maxsize > 2**32)
#
ffi = FFI(backend=self.Backend())
ffi.cdef("""
BOOL QueryPerformanceFrequency(LONGLONG *lpFrequency);
""")
m = ffi.dlopen("Kernel32.dll")
tp = ffi.typeof(m.QueryPerformanceFrequency)
assert str(tp) == "<ctype 'int(*)(long long *)'>"
#
ffi = FFI(backend=self.Backend())
ffi.cdef("""
BOOL __cdecl QueryPerformanceFrequency(LONGLONG *lpFrequency);
""")
m = ffi.dlopen("Kernel32.dll")
tpc = ffi.typeof(m.QueryPerformanceFrequency)
assert tpc is tp
#
ffi = FFI(backend=self.Backend())
ffi.cdef("""
BOOL WINAPI QueryPerformanceFrequency(LONGLONG *lpFrequency);
""")
m = ffi.dlopen("Kernel32.dll")
tps = ffi.typeof(m.QueryPerformanceFrequency)
if win64:
assert tps is tpc
else:
assert tps is not tpc
assert str(tps) == "<ctype 'int(__stdcall *)(long long *)'>"
#
ffi = FFI(backend=self.Backend())
ffi.cdef("typedef int (__cdecl *fnc_t)(int);")
ffi.cdef("typedef int (__stdcall *fns_t)(int);")
tpc = ffi.typeof("fnc_t")
tps = ffi.typeof("fns_t")
assert str(tpc) == "<ctype 'int(*)(int)'>"
if win64:
assert tps is tpc
else:
assert str(tps) == "<ctype 'int(__stdcall *)(int)'>"
#
fnc = ffi.cast("fnc_t", 0)
fns = ffi.cast("fns_t", 0)
ffi.new("fnc_t[]", [fnc])
if not win64:
py.test.raises(TypeError, ffi.new, "fnc_t[]", [fns])
py.test.raises(TypeError, ffi.new, "fns_t[]", [fnc])
ffi.new("fns_t[]", [fns])
def test_include_3():
ffi1 = FFI()
ffi1.cdef("typedef short sshort_t;")
verify(ffi1, "test_include_3_parent", "typedef short sshort_t;")
ffi = FFI()
ffi.include(ffi1)
ffi.cdef("sshort_t ff3(sshort_t);")
lib = verify(ffi, "test_include_3",
"typedef short sshort_t; //usually from a #include\n"
"sshort_t ff3(sshort_t x) { return x + 42; }")
assert lib.ff3(10) == 52
assert ffi.typeof(ffi.cast("sshort_t", 42)) is ffi.typeof("short")
assert ffi1.typeof("sshort_t") is ffi.typeof("sshort_t")
def test_verify_anonymous_enum_with_typedef():
ffi = FFI()
ffi.cdef("typedef enum { AA, ... } e1;")
lib = verify(ffi, 'test_verify_anonymous_enum_with_typedef1',
"typedef enum { BB, CC, AA } e1;")
assert lib.AA == 2
assert ffi.sizeof("e1") == ffi.sizeof("int")
assert repr(ffi.cast("e1", 2)) == "<cdata 'e1' 2: AA>"
#
ffi = FFI()
ffi.cdef("typedef enum { AA=%d } e1;" % sys.maxsize)
lib = verify(ffi, 'test_verify_anonymous_enum_with_typedef2',
"typedef enum { AA=%d } e1;" % sys.maxsize)
assert lib.AA == sys.maxsize
assert ffi.sizeof("e1") == ffi.sizeof("long")
def test_verify_enum():
ffi = FFI()
ffi.cdef("""enum e1 { B1, A1, ... }; enum e2 { B2, A2, ... };""")
lib = verify(ffi, 'test_verify_enum',
"enum e1 { A1, B1, C1=%d };" % sys.maxsize +
"enum e2 { A2, B2, C2 };")
ffi.typeof("enum e1")
ffi.typeof("enum e2")
assert lib.A1 == 0
assert lib.B1 == 1
assert lib.A2 == 0
assert lib.B2 == 1
assert ffi.sizeof("enum e1") == ffi.sizeof("long")
assert ffi.sizeof("enum e2") == ffi.sizeof("int")
assert repr(ffi.cast("enum e1", 0)) == "<cdata 'enum e1' 0: A1>"
def paste(self, im, box=None):
"""
Paste a PIL image into the photo image. Note that this can
be very slow if the photo image is displayed.
:param im: A PIL image. The size must match the target region. If the
mode does not match, the image is converted to the mode of
the bitmap image.
:param box: A 4-tuple defining the left, upper, right, and lower pixel
coordinate. See :ref:`coordinate-system`. If None is given
instead of a tuple, all of the image is assumed.
"""
# convert to blittable
im.load()
image = im.im
if image.isblock() and im.mode == self.__mode:
block = image
else:
block = image.new_block(self.__mode, im.size)
image.convert2(block, image) # convert directly between buffers
tk = self.__photo.tk
try:
tk.call("PyImagingPhoto", self.__photo, block.id)
except tkinter.TclError:
# activate Tkinter hook
try:
from . import _imagingtk
try:
if hasattr(tk, 'interp'):
# Required for PyPy, which always has CFFI installed
from cffi import FFI
ffi = FFI()
# PyPy is using an FFI CDATA element
# (Pdb) self.tk.interp
# <cdata 'Tcl_Interp *' 0x3061b50>
_imagingtk.tkinit(
int(ffi.cast("uintptr_t", tk.interp)), 1)
else:
_imagingtk.tkinit(tk.interpaddr(), 1)
except AttributeError:
_imagingtk.tkinit(id(tk), 0)
tk.call("PyImagingPhoto", self.__photo, block.id)
except (ImportError, AttributeError, tkinter.TclError):
raise # configuration problem; cannot attach to Tkinter
def test_include_7():
ffi1 = FFI()
ffi1.cdef("typedef ... mystruct_t;\n"
"int ff7b(mystruct_t *);")
verify(ffi1, "test_include_7_parent",
"typedef struct { int x; } mystruct_t;\n"
"int ff7b(mystruct_t *p) { return p->x; }")
ffi = FFI()
ffi.include(ffi1)
ffi.cdef("mystruct_t *ff7(void);")
lib = verify(ffi, "test_include_7",
"typedef struct { int x; } mystruct_t; //usually from a #include\n"
"static mystruct_t result_struct = { 42 };"
"mystruct_t *ff7(void) { return &result_struct; }")
p = lib.ff7()
assert ffi.cast("int *", p)[0] == 42
assert lib.ff7b(p) == 42
def main():
"""Program main."""
ffi = FFI()
ffi.cdef("""
double integral_to_infinite(double a, double b, double E);
double to_degrees(double radians);
""")
# C = ffi.dlopen(None)
lib = ffi.verify("""
#include "tetaQuad.h"
""", include_dirs=["."], extra_compile_args=["-O3"])
var_e = ffi.cast("double", 0.1)
var_a = ffi.cast("double", 0.0)
points_x = []
points_y = []
points_y_a = []
print("Calculus may take some time...")
for var_b in range_f(0.0, 100, 0.5):
points_x.append(var_b)
rad = lib.integral_to_infinite(var_a, var_b, var_e)
theta = lib.to_degrees(abs(rad))
analytic = degrees(analytic_evaluation(var_b, var_e))
print("b = {}\ttheta = {}\tanalytic = {}".format(
float(var_b), theta, analytic))
points_y.append(theta)
points_y_a.append(analytic)
plt.ylabel('teta')
plt.xlabel('b')
# plt.plot(points_x, points_y, 'r--')
plt.plot(points_x, points_y, 'ro')
plt.plot(points_x, points_y_a, 'b--')
# plt.plot(points_x, points_y_a, 'bo')
red_line = mpatches.Patch(color='red', label='theta value')
blue_line = mpatches.Patch(color='blue', label='analytic value')
plt.legend(handles=[red_line, blue_line])
plt.grid(True)
plt.show()
def verify_rnn_forward(net):
'''Test network with given input data on both darknet and tvm'''
def get_darknet_network_predict(net, data):
return LIB.network_predict(net, data)
from cffi import FFI
ffi = FFI()
np_arr = np.zeros([1, net.inputs], dtype='float32')
np_arr[0, 84] = 1
cffi_arr = ffi.cast('float*', np_arr.ctypes.data)
tvm_out = _get_tvm_output(net, np_arr)[0]
darknet_output = get_darknet_network_predict(net, cffi_arr)
darknet_out = np.zeros(net.outputs, dtype='float32')
for i in range(net.outputs):
darknet_out[i] = darknet_output[i]
last_layer = net.layers[net.n-1]
darknet_outshape = (last_layer.batch, last_layer.outputs)
darknet_out = darknet_out.reshape(darknet_outshape)
tvm.testing.assert_allclose(darknet_out, tvm_out, rtol=1e-4, atol=1e-4)
def c_charptrptr_as_string_list(ffi: FFI, ptr: CffiData,
size: int) -> List[str]:
"""Convert if possible a cffi pointer to a C data array char** , into a list of python strings.
Args:
ffi (FFI): FFI instance wrapping the native compilation module owning the native memory
ptr (CffiData): cffi pointer (FFI.CData)
size (int): number of character strings in the char** pointer
Raises:
RuntimeError: conversion is not supported
Returns:
List[str]: converted data
"""
# TODO check type
strings = ffi.cast('char*[%d]' % (size, ), ptr)
res = [as_string(ffi.string(strings[i])) for i in range(size)]
return res
def test_struct_array_no_length(self):
ffi = FFI()
ffi.cdef("struct foo_s { int x; int a[]; };")
p = ffi.new("struct foo_s *", [100, [200, 300, 400]])
assert p.x == 100
assert ffi.typeof(p.a) is ffi.typeof("int[]")
assert len(p.a) == 3 # length recorded
assert p.a[0] == 200
assert p.a[1] == 300
assert p.a[2] == 400
assert list(p.a) == [200, 300, 400]
q = ffi.cast("struct foo_s *", p)
assert q.x == 100
assert ffi.typeof(q.a) is ffi.typeof("int *") # no length recorded
py.test.raises(TypeError, len, q.a)
assert q.a[0] == 200
assert q.a[1] == 300
assert q.a[2] == 400
py.test.raises(TypeError, list, q.a)
def test_struct_array_no_length(self):
ffi = FFI()
ffi.cdef("struct foo_s { int x; int a[]; };")
p = ffi.new("struct foo_s *", [100, [200, 300, 400]])
assert p.x == 100
assert ffi.typeof(p.a) is ffi.typeof("int[]")
assert len(p.a) == 3 # length recorded
assert p.a[0] == 200
assert p.a[1] == 300
assert p.a[2] == 400
assert list(p.a) == [200, 300, 400]
q = ffi.cast("struct foo_s *", p)
assert q.x == 100
assert ffi.typeof(q.a) is ffi.typeof("int *") # no length recorded
py.test.raises(TypeError, len, q.a)
assert q.a[0] == 200
assert q.a[1] == 300
assert q.a[2] == 400
py.test.raises(TypeError, list, q.a)
def dVdg_wrapper(xp,
yp,
weights,
q_true,
t_true,
hdx_p,
hdy_p,
hnd_raw_p,
test=False):
if test:
from test_constants import array_length_test, const_length_test, big_array_length_test
from _geometry_test.lib import dVdg_function_c
array_length = array_length_test
const_length = const_length_test
big_array_length = big_array_length_test
else:
from constants import array_length_real, const_length_real, big_array_length_real
from _geometry2.lib import dVdg_function_c
array_length = array_length_real
const_length = const_length_real
big_array_length = big_array_length_real
ffi = FFI()
xp_c = deepcopy(xp)
yp_c = deepcopy(yp)
xp_p = ffi.cast("double*", xp_c.__array_interface__['data'][0])
yp_p = ffi.cast("double*", yp_c.__array_interface__['data'][0])
q_truec = q_true
q_truep = ffi.new('double[4]', q_truec.tolist())
t_true_c = t_true
t_true_p = ffi.new('double[3]', t_true_c.tolist())
weights_c = deepcopy(weights)
weights_p = ffi.cast('double*', weights_c.__array_interface__['data'][0])
r_xc = np.zeros(const_length)
r_xp = ffi.cast('double*', r_xc.__array_interface__['data'][0])
r_yc = np.zeros(const_length)
r_yp = ffi.cast('double*', r_yc.__array_interface__['data'][0])
hnd_c = np.zeros(array_length)
hnd_p = ffi.cast('double*', hnd_c.__array_interface__['data'][0])
dVdg_c = np.zeros(array_length)
dVdg_p = ffi.cast('double*', dVdg_c.__array_interface__['data'][0])
Hnd_R_c = np.zeros(array_length)
Hnd_R_p = ffi.cast('double*', Hnd_R_c.__array_interface__['data'][0])
V_c = dVdg_function_c(q_truep, t_true_p, weights_p, xp_p, yp_p, hdx_p,
hdy_p, hnd_raw_p, dVdg_p, r_xp, r_yp)
return np.array([V_c]), dVdg_c, r_xc, r_yc
def _test_rnn_network(net, states):
"""Test network with given input data on both darknet and tvm"""
def get_darknet_network_predict(net, data):
return LIB.network_predict(net, data)
from cffi import FFI
ffi = FFI()
np_arr = np.zeros([1, net.inputs], dtype="float32")
np_arr[0, 2] = 1
cffi_arr = ffi.cast("float*", np_arr.ctypes.data)
tvm_out = _get_tvm_output(net, np_arr, states=states)[0]
darknet_output = get_darknet_network_predict(net, cffi_arr)
darknet_out = np.zeros(net.outputs, dtype="float32")
for i in range(net.outputs):
darknet_out[i] = darknet_output[i]
last_layer = net.layers[net.n - 1]
darknet_outshape = (last_layer.batch, last_layer.outputs)
darknet_out = darknet_out.reshape(darknet_outshape)
tvm.testing.assert_allclose(darknet_out, tvm_out, rtol=1e-4, atol=1e-4)
def test_variable_of_unknown_size():
ffi = FFI()
ffi.cdef("""
typedef ... opaque_t;
opaque_t globvar;
""")
lib = verify(ffi, 'test_variable_of_unknown_size', """
typedef char opaque_t[6];
opaque_t globvar = "hello";
""")
# can't read or write it at all
e = py.test.raises(TypeError, getattr, lib, 'globvar')
assert str(e.value) in ["cdata 'opaque_t' is opaque",
"'opaque_t' is opaque or not completed yet"] #pypy
e = py.test.raises(TypeError, setattr, lib, 'globvar', [])
assert str(e.value) in ["'opaque_t' is opaque",
"'opaque_t' is opaque or not completed yet"] #pypy
# but we can get its address
p = ffi.addressof(lib, 'globvar')
assert ffi.typeof(p) == ffi.typeof('opaque_t *')
assert ffi.string(ffi.cast("char *", p), 8) == b"hello"
def test_variable_of_unknown_size():
ffi = FFI()
ffi.cdef("""
typedef ... opaque_t;
opaque_t globvar;
""")
lib = verify(ffi, 'test_variable_of_unknown_size', """
typedef char opaque_t[6];
opaque_t globvar = "hello";
""")
# can't read or write it at all
e = py.test.raises(TypeError, getattr, lib, 'globvar')
assert str(e.value) in ["cdata 'opaque_t' is opaque",
"'opaque_t' is opaque or not completed yet"] #pypy
e = py.test.raises(TypeError, setattr, lib, 'globvar', [])
assert str(e.value) in ["'opaque_t' is opaque",
"'opaque_t' is opaque or not completed yet"] #pypy
# but we can get its address
p = ffi.addressof(lib, 'globvar')
assert ffi.typeof(p) == ffi.typeof('opaque_t *')
assert ffi.string(ffi.cast("char *", p), 8) == b"hello"
class LibHMuMu:
def __init__(self, libpath=os.path.dirname(os.path.realpath(__file__)) + "/libhmm.so"):
self.ffi = FFI()
self.ffi.cdef("""
void* new_roccor(char* filename);
void roccor_kScaleDT(void* rc, float* out, int n_elem, int* charges, float* pt, float* eta, float* phi, int s, int m);
void roccor_kSpreadMC_or_kSmearMC(void* rc, float* out, int n_elem,
int* charges, float* pt, float* eta, float* phi,
float* genpt, int* tracklayers, float* rand, int s, int m);
void* new_LeptonEfficiencyCorrector(int n, const char** file, const char** histo, float* weights);
void LeptonEfficiencyCorrector_getSF(void* c, float* out, int n, int* pdgid, float* pt, float* eta);
const void* new_gbr(const char* weightfile);
int gbr_get_nvariables(const void* gbr);
void gbr_eval(const void* gbr, float* out, int nev, int nfeatures, float* inputs_matrix);
void csangles_eval(float* out_theta, float* out_phi, int nev, float* pt1, float* eta1, float* phi1, float* mass1, float* pt2, float* eta2, float* phi2, float* mass2, int* charges);
void mllErr_eval(float* out_err, int nev, float* pt1, float* eta1, float* phi1, float* mass1, float* pt2, float* eta2, float* phi2, float* mass2, float* Err1, float* Err2);
""")
self.libhmm = self.ffi.dlopen(libpath)
self.new_roccor = self.libhmm.new_roccor
self.roccor_kScaleDT = self.libhmm.roccor_kScaleDT
self.roccor_kSpreadMC_or_kSmearMC = self.libhmm.roccor_kSpreadMC_or_kSmearMC
self.new_LeptonEfficiencyCorrector = self.libhmm.new_LeptonEfficiencyCorrector
self.LeptonEfficiencyCorrector_getSF = self.libhmm.LeptonEfficiencyCorrector_getSF
self.new_gbr = self.libhmm.new_gbr
self.gbr_get_nvariables = self.libhmm.gbr_get_nvariables
self.gbr_eval = self.libhmm.gbr_eval
self.csangles_eval = self.libhmm.csangles_eval
self.mllErr_eval = self.libhmm.mllErr_eval
def cast_as(self, dtype_string, arr):
return self.ffi.cast(dtype_string, arr.ctypes.data)
def search_neighbors(context,
ref_indices=None,
coordinates=None,
view_vectors=None,
angles_deg=None,
naive=False):
ffi = FFI()
if ref_indices is None:
num_indices = len(coordinates)
x, y = zip(*coordinates)
x_np = np.array(x)
x_p = ffi.cast("double *", x_np.ctypes.data)
y_np = np.array(y)
y_p = ffi.cast("double *", y_np.ctypes.data)
else:
num_indices = len(ref_indices)
indices_np = np.zeros(num_indices, dtype=np.int32)
indices_p = ffi.cast("int *", indices_np.ctypes.data)
if (view_vectors is None and angles_deg is None):
use_angles = False
vx_p = ffi.new("double *")
vy_p = ffi.new("double *")
angles_deg_p = ffi.new("double *")
else:
use_angles = True
vx, vy = zip(*view_vectors)
vx_np = np.array(vx)
vx_p = ffi.cast("double *", vx_np.ctypes.data)
vy_np = np.array(vy)
vy_p = ffi.cast("double *", vy_np.ctypes.data)
angles_deg_np = np.array(angles_deg)
angles_deg_p = ffi.cast("double *", angles_deg_np.ctypes.data)
if ref_indices is None:
_lib.search_neighbors_by_coordinates(context, num_indices, indices_p,
x_p, y_p, use_angles, vx_p, vy_p,
angles_deg_p, naive)
else:
_lib.search_neighbor_by_indices(context, num_indices, indices_p,
ref_indices, use_angles, vx_p, vy_p,
angles_deg_p, naive)
return indices_np.tolist()
def main_simple():
""" Pass a numpy array to a C function and get a numpy array back out
Cleaned up version, from:
http://stackoverflow.com/questions/16276268/how-to-pass-a-numpy-array-into-a-cffi-function-and-how-to-get-one-back-out
"""
ffi = FFI()
ffi.cdef("void copy(float *in, float *out, int len);")
C = ffi.dlopen("./libcopy.so")
# Create float32 numpy array
a = 42 * np.ones(16, dtype=np.float32)
b = np.zeros_like(a)
# Cast data to readable format by C
pa = ffi.cast("float *", a.ctypes.data)
pb = ffi.cast("float *", b.ctypes.data)
C.copy(pa, pb, len(a))
# pb is C version of b, so values of b still changed
print(b)
return
def main_experimental():
""" Pass a numpy array to a C function and get a numpy array back out
Experimental version (not guaranteed to work)
"""
ffi = FFI()
ffi.cdef("void copy(float *in, float *out, int len);")
#C = ffi.dlopen("./libcopy.so")
C = ffi.verify("""#include "copy.h" """,libraries=['libcopy'])
# Create float32 numpy array
a = 42 * np.ones(16, dtype=np.float32)
b = np.zeros_like(a)
# Cast data to readable format by C
pa = ffi.cast("float *", a.ctypes.data)
pb = ffi.cast("float *", b.ctypes.data)
C.copy(pa, pb, len(a))
# pb is C version of b, so values of b still changed
print(b)
return
def as_np_array_double(ffi: FFI,
ptr: CffiData,
size: int,
shallow: bool = False) -> np.ndarray:
"""Convert if possible a cffi pointer to a C data array, into a numpy array of double precision floats.
Args:
ffi (FFI): FFI instance wrapping the native compilation module owning the native memory
ptr (CffiData): cffi pointer (FFI.CData)
size (int): array size
shallow (bool): If true the array points directly to native data array. Defaults to False.
Raises:
RuntimeError: conversion is not supported
Returns:
np.ndarray: converted data
"""
res = np.frombuffer(ffi.buffer(ffi.cast('double[%d]' % (size, ), ptr)))
if shallow:
return res
else:
return res.copy()
def test_ffi_new_allocator_2(self):
ffi = FFI(backend=self.Backend())
seen = []
def myalloc(size):
seen.append(size)
return ffi.new("char[]", b"X" * size)
def myfree(raw):
seen.append(raw)
alloc1 = ffi.new_allocator(myalloc, myfree)
alloc2 = ffi.new_allocator(alloc=myalloc,
free=myfree,
should_clear_after_alloc=False)
p1 = alloc1("int[10]")
p2 = alloc2("int[]", 10)
assert seen == [40, 40]
assert ffi.typeof(p1) == ffi.typeof("int[10]")
assert ffi.sizeof(p1) == 40
assert ffi.typeof(p2) == ffi.typeof("int[]")
assert ffi.sizeof(p2) == 40
assert p1[5] == 0
assert p2[6] == ord('X') * 0x01010101
raw1 = ffi.cast("char *", p1)
raw2 = ffi.cast("char *", p2)
del p1, p2
retries = 0
while len(seen) != 4:
retries += 1
assert retries <= 5
import gc
gc.collect()
assert seen == [40, 40, raw1, raw2]
assert repr(seen[2]) == "<cdata 'char[]' owning 41 bytes>"
assert repr(seen[3]) == "<cdata 'char[]' owning 41 bytes>"
def as_c_double_array(ffi: FFI,
data: np.ndarray,
shallow: bool = False) -> OwningCffiNativeHandle:
if isinstance(data, list):
data = np.asfarray(data)
shallow = False
elif isinstance(data, xr.DataArray):
data = data.values
# shallow = False # really needed??
elif not isinstance(data, np.ndarray):
raise TypeError(
"Conversion to a c array of double requires list or np array as input"
)
if len(data.shape) > 1:
data = data.squeeze()
shallow = False
if len(data.shape) > 1:
raise TypeError(
"Conversion to a double* array: input data must be of dimension one, and the python array cannot be squeezed to dimension one"
)
if not (data.dtype == np.float or data.dtype == np.float64 or data.dtype
== float or data.dtype == np.double or data.dtype == np.float_):
# https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
# TODO: is this wise to override the shallow parameter
shallow = False
data = data.astype(np.float64)
if shallow and data.flags['C_CONTIGUOUS']:
native_d = ffi.cast("double *", data.ctypes.data)
else:
native_d = new_double_array(ffi, data.shape[0])
if not data.flags['C_CONTIGUOUS']:
data_c = np.ascontiguousarray(data)
else:
data_c = data
ffi.buffer(native_d)[:] = data_c
return OwningCffiNativeHandle(native_d)
def create_coordinate_map(o):
"""Return a compiled UFC coordinate_mapping object"""
try:
# Create a compiled coordinate map from an object with the
# ufl_mesh attribute
cmap_ptr = jit.ffc_jit(o.ufl_domain())
except AttributeError:
# FIXME: It would be good to avoid the type check, but ffc_jit
# supports other objects so we could get, e.g., a compiled
# finite element
if isinstance(o, ufl.domain.Mesh):
cmap_ptr = jit.ffc_jit(o)
else:
raise TypeError(
"Cannot create coordinate map from an object of type: {}".format(type(o)))
except Exception:
print("Failed to create compiled coordinate map")
raise
# Wrap compiled coordinate map and return
ffi = FFI()
ufc_cmap = fem.dofmap.make_ufc_coordinate_mapping(ffi.cast("uintptr_t", cmap_ptr))
return cpp.fem.CoordinateMapping(ufc_cmap)
def check(self, source, expected_ofs_y, expected_align, expected_size):
# NOTE: 'expected_*' is the numbers expected from GCC.
# The numbers expected from MSVC are not explicitly written
# in this file, and will just be taken from the compiler.
ffi = FFI()
ffi.cdef("struct s1 { %s };" % source)
ctype = ffi.typeof("struct s1")
# verify the information with gcc
ffi1 = FFI()
ffi1.cdef("""
static const int Gofs_y, Galign, Gsize;
struct s1 *try_with_value(int fieldnum, long long value);
""")
fnames = [
name for name, cfield in ctype.fields
if name and cfield.bitsize > 0
]
setters = [
'case %d: s.%s = value; break;' % iname
for iname in enumerate(fnames)
]
lib = ffi1.verify("""
struct s1 { %s };
struct sa { char a; struct s1 b; };
#define Gofs_y offsetof(struct s1, y)
#define Galign offsetof(struct sa, b)
#define Gsize sizeof(struct s1)
struct s1 *try_with_value(int fieldnum, long long value)
{
static struct s1 s;
memset(&s, 0, sizeof(s));
switch (fieldnum) { %s }
return &s;
}
""" % (source, ' '.join(setters)))
if sys.platform == 'win32':
expected_ofs_y = lib.Gofs_y
expected_align = lib.Galign
expected_size = lib.Gsize
else:
assert (lib.Gofs_y, lib.Galign,
lib.Gsize) == (expected_ofs_y, expected_align,
expected_size)
# the real test follows
assert ffi.offsetof("struct s1", "y") == expected_ofs_y
assert ffi.alignof("struct s1") == expected_align
assert ffi.sizeof("struct s1") == expected_size
# compare the actual storage of the two
for name, cfield in ctype.fields:
if cfield.bitsize < 0 or not name:
continue
if int(ffi.cast(cfield.type, -1)) == -1: # signed
min_value = -(1 << (cfield.bitsize - 1))
max_value = (1 << (cfield.bitsize - 1)) - 1
else:
min_value = 0
max_value = (1 << cfield.bitsize) - 1
for t in [
1, 2, 4, 8, 16, 128, 2813, 89728, 981729, -1, -2, -4, -8,
-16, -128, -2813, -89728, -981729
]:
if min_value <= t <= max_value:
self._fieldcheck(ffi, lib, fnames, name, t)
lib = ffi.dlopen('./distributions.dll')
elif os.path.exists('./libdistributions.so'):
lib = ffi.dlopen('./libdistributions.so')
else:
raise RuntimeError('Required DLL/so file was not found.')
ffi.cdef("""
double random_gauss_zig(void *brng_state);
""")
x = Xoroshiro128()
xffi = x.cffi
brng = xffi.brng
random_gauss_zig = lib.random_gauss_zig
def normals(n, brng):
out = np.empty(n)
for i in range(n):
out[i] = random_gauss_zig(brng)
return out
normalsj = nb.jit(normals, nopython=True)
# Numba requires a memory address for void *
# Can also get address from x.ctypes.brng.value
brng_address = int(ffi.cast('uintptr_t', brng))
norm = normalsj(1000, brng_address)
elif os.path.exists('./libdistributions.so'):
lib = ffi.dlopen('./libdistributions.so')
else:
raise RuntimeError('Required DLL/so file was not found.')
ffi.cdef("""
double random_standard_normal(void *bitgen_state);
""")
x = PCG64()
xffi = x.cffi
bit_generator = xffi.bit_generator
random_standard_normal = lib.random_standard_normal
def normals(n, bit_generator):
out = np.empty(n)
for i in range(n):
out[i] = random_standard_normal(bit_generator)
return out
normalsj = nb.jit(normals, nopython=True)
# Numba requires a memory address for void *
# Can also get address from x.ctypes.bit_generator.value
bit_generator_address = int(ffi.cast('uintptr_t', bit_generator))
norm = normalsj(1000, bit_generator_address)
print(norm[:12])
class _PcapFfi(object):
"""
This class represents the low-level interface to the libpcap library.
It encapsulates all the cffi calls and C/Python conversions, as well
as translation of errors and error codes to PcapExceptions. It is
intended to be used as a singleton class through the PcapDumper
and PcapLiveDevice classes, below.
"""
_instance = None
__slots__ = ['_ffi', '_libpcap', '_interfaces', '_windows']
def __init__(self):
"""
Assumption: this class is instantiated once in the main thread before
any other threads have a chance to try instantiating it.
"""
if _PcapFfi._instance:
raise Exception("Can't initialize this class more than once!")
_PcapFfi._instance = self
self._windows = False
self._ffi = FFI()
self._ffi.cdef(cc, override=True)
self._ffi.cdef(cc_packed, override=True, packed=1)
if sys.platform == 'darwin':
libname = 'libpcap.dylib'
elif "win" in sys.platform[:3]:
libname = 'wpcap.dll' # winpcap
self._windows = True
else:
# if not macOS (darwin) or windows, assume we're on
# some unix-based system and try for libpcap.so
libname = 'libpcap.so'
try:
self._libpcap = self._ffi.dlopen(libname)
except Exception as e:
raise PcapException("Error opening libpcap: {}".format(e))
self._interfaces = []
self.discoverdevs()
@staticmethod
def instance():
if not _PcapFfi._instance:
_PcapFfi._instance = _PcapFfi()
return _PcapFfi._instance
@property
def version(self):
return self._ffi.string(self._libpcap.pcap_lib_version())
def discoverdevs(self):
"""
Find all the pcap-eligible devices on the local system.
"""
if len(self._interfaces):
raise PcapException("Device discovery should only be done once.")
ppintf = self._ffi.new("pcap_if_t * *")
errbuf = self._ffi.new("char []", 128)
rv = self._libpcap.pcap_findalldevs(ppintf, errbuf)
if rv:
raise PcapException("pcap_findalldevs returned failure: {}".format(
self._ffi.string(errbuf)))
pintf = ppintf[0]
tmp = pintf
pindex = 0
while tmp != self._ffi.NULL:
xname = self._ffi.string(
tmp.name) # "internal name"; still stored as bytes object
xname = xname.decode('ascii', 'ignore')
if self._windows:
ext_name = "port{}".format(pindex)
else:
ext_name = xname
pindex += 1
if tmp.description == self._ffi.NULL:
xdesc = ext_name
else:
xdesc = self._ffi.string(tmp.description)
xdesc = xdesc.decode('ascii', 'ignore')
# NB: on WinPcap, only loop flag is set
isloop = (tmp.flags & 0x1) == 0x1
isup = (tmp.flags & 0x2) == 0x2
isrunning = (tmp.flags & 0x4) == 0x4
xif = PcapInterface(ext_name, xname, xdesc, isloop, isup,
isrunning)
self._interfaces.append(xif)
tmp = tmp.next
self._libpcap.pcap_freealldevs(pintf)
@property
def devices(self):
return self._interfaces
@property
def lib(self):
return self._libpcap
@property
def ffi(self):
return self._ffi
def _process_packet(self, xdev, header, packet, nroots):
# MPLS header
mpls = self._ffi.new("union mpls *")
# IP header
iph = self._ffi.new("struct nfstream_iphdr *")
# IPv6 header
iph6 = self._ffi.new("struct nfstream_ipv6hdr *")
# lengths and offsets
eth_offset = 0
radio_len = 0
fc = 0
type = 0
wifi_len = 0
pyld_eth_len = 0
check = 0
ip_offset = 0
ip_len = 0
frag_off = 0
vlan_id = 0
proto = 0
time = 0
time = (header.tv_sec *
TICK_RESOLUTION) + (header.tv_usec /
(1000000 / TICK_RESOLUTION))
datalink_type = self._libpcap.pcap_datalink(xdev)
datalink_check = True
while datalink_check:
datalink_check = False
if Dlt(datalink_type) == Dlt.DLT_NULL:
tmp_dlt_null = self._ffi.cast('struct pp_32 *',
packet + eth_offset)
if int(ntohs(tmp_dlt_null.value)) == 2:
type = 0x0800
else:
type = 0x86dd
ip_offset = 4 + eth_offset
elif Dlt(
datalink_type
) == Dlt.DLT_PPP_SERIAL: # Cisco PPP in HDLC - like framing - 50
chdlc = self._ffi.cast('struct nfstream_chdlc *',
packet + eth_offset)
ip_offset = self._ffi.sizeof(
'struct nfstream_chdlc') # CHDLC_OFF = 4
type = ntohs(chdlc.proto_code)
elif (Dlt(datalink_type) == Dlt.DLT_C_HDLC) or (
Dlt(datalink_type) == Dlt.DLT_PPP): # Cisco PPP - 9 or 104
chdlc = self._ffi.cast('struct nfstream_chdlc *',
packet + eth_offset) # CHDLC_OFF = 4
ip_offset = self._ffi.sizeof(
'struct nfstream_chdlc') # CHDLC_OFF = 4
type = ntohs(chdlc.proto_code)
elif Dlt(datalink_type
) == Dlt.DLT_EN10MB: # IEEE 802.3 Ethernet - 1 */
ethernet = self._ffi.cast('struct nfstream_ethhdr *',
packet + eth_offset)
ip_offset = self._ffi.sizeof(
'struct nfstream_ethhdr') + eth_offset
check = ntohs(ethernet.h_proto)
if check <= 1500:
pyld_eth_len = check
elif check >= 1536:
type = check
if pyld_eth_len != 0:
llc = self._ffi.cast('struct nfstream_llc_header_snap *',
packet + ip_offset)
if (llc.dsap == 0xaa) or (
llc.ssap
== 0xaa): # check for LLC layer with SNAP ext
type = llc.snap.proto_ID
ip_offset += 8
elif (llc.dsap == 0x42) or (llc.ssap
== 0x42): # No SNAP ext
return None
elif Dlt(datalink_type
) == Dlt.DLT_LINUX_SLL: # Linux Cooked Capture - 113
type = (packet[eth_offset + 14] << 8) + packet[eth_offset + 15]
ip_offset = 16 + eth_offset
elif Dlt(
datalink_type
) == Dlt.DLT_IEEE802_11_RADIO: # Radiotap link - layer - 127
radiotap = self._ffi.cast('struct nfstream_radiotap_header *',
packet + eth_offset)
radio_len = radiotap.len
if (radiotap.flags & 0x50) == 0x50: # Check Bad FCS presence
return None
# Calculate 802.11 header length(variable)
wifi = self._ffi.cast('struct nfstream_wifi_header *',
packet + (eth_offset + radio_len))
fc = wifi.fc
# Check wifi data presence
if fcf_type(fc) == 0x2:
if (fcf_to_ds(fc) and fcf_from_ds(fc) == 0x0) or (
fcf_to_ds(fc) == 0x0 and fcf_from_ds(fc)):
wifi_len = 26 # + 4 byte fcs
else:
pass
# Check ether_type from LLC
llc = self._ffi.cast(
'struct nfstream_llc_header_snap *',
packet + (eth_offset + wifi_len + radio_len))
if llc.dsap == 0xaa:
type = ntohs(llc.snap.proto_ID)
# Set IP header offset
ip_offset = wifi_len + radio_len + self._ffi.sizeof(
'struct nfstream_llc_header_snap') + eth_offset
elif Dlt(datalink_type) == Dlt.DLT_RAW:
ip_offset = 0
eth_offset = 0
else:
return None
ether_type_check = True
while ether_type_check:
ether_type_check = False
if type == 0x8100:
vlan_id = ((packet[ip_offset] << 8) +
packet[ip_offset + 1]) & 0xFFF
type = (packet[ip_offset + 2] << 8) + packet[ip_offset + 3]
ip_offset += 4
while type == 0x8100 and ip_offset < header.caplen: # Double tagging for 802.1Q
vlan_id = ((packet[ip_offset] << 8) +
packet[ip_offset + 1]) & 0xFFF
type = (
packet[ip_offset + 2] << 8) + packet[ip_offset + 3]
ip_offset += 4
ether_type_check = True
elif (type == 0x8847) or (type == 0x8848):
tmp_u32 = self._ffi.cast('struct pp_32 *',
packet + ip_offset)
mpls.u32 = int(ntohl(tmp_u32.value))
type = 0x0800
ip_offset += 4
while not mpls.mpls.s:
tmp_u32_loop = self._ffi.cast('struct pp_32 *',
packet + ip_offset)
mpls.u32 = int(ntohl(tmp_u32_loop.value))
ip_offset += 4
ether_type_check = True
elif type == 0x8864:
type = 0x0800
ip_offset += 8
ether_type_check = True
else:
pass
ip_check = True
while ip_check:
ip_check = False
# Check and set IP header size and total packet length
iph = self._ffi.cast('struct nfstream_iphdr *',
packet + ip_offset)
# Just work on Ethernet packets that contain IP
if (type == 0x0800) and (header.caplen >= ip_offset):
frag_off = ntohs(iph.frag_off)
if header.caplen < header.len:
pass
if iph.version == 4:
ip_len = iph.ihl * 4
iph6 = self._ffi.NULL
if iph.protocol == 41: # IPPROTO_IPV6
ip_offset += ip_len
ip_check = True
if (frag_off & 0x1FFF) != 0:
return None
elif iph.version == 6:
iph6 = self._ffi.cast('struct nfstream_ipv6hdr *',
packet + ip_offset)
ip_len = self._ffi.sizeof('struct nfstream_ipv6hdr')
if iph6.ip6_hdr.ip6_un1_nxt == 60: # IPv6 destination option
options = self._ffi.cast('uint8_t *',
packet + (ip_offset + ip_len))
ip_len += 8 * (options[1] + 1)
iph = self._ffi.NULL
else:
return None
l4_offset = 0
ipsize = 0
src_addr = 0
dst_addr = 0
l4_packet_len = 0
version = 0
nfstream_hash = 0
if iph6 == self._ffi.NULL:
version = 4
l4_packet_len = ntohs(iph.tot_len) - (iph.ihl * 4)
ipsize = header.caplen - ip_offset
proto = iph.protocol
src_addr = ntohl(iph.saddr)
dst_addr = ntohl(iph.daddr)
nfstream_hash += iph.saddr + iph.daddr + proto + vlan_id
else:
version = 6
src_addr = ntohl(iph6.ip6_src.u6_addr.u6_addr32[0]) << 96 | ntohl(
iph6.ip6_src.u6_addr.u6_addr32[1]) << 64 | ntohl(
iph6.ip6_src.u6_addr.u6_addr32[2]) << 32 | ntohl(
iph6.ip6_src.u6_addr.u6_addr32[3])
dst_addr = ntohl(iph6.ip6_dst.u6_addr.u6_addr32[0]) << 96 | ntohl(
iph6.ip6_dst.u6_addr.u6_addr32[1]) << 64 | ntohl(
iph6.ip6_dst.u6_addr.u6_addr32[2]) << 32 | ntohl(
iph6.ip6_dst.u6_addr.u6_addr32[3])
proto = iph6.ip6_hdr.ip6_un1_nxt
if proto == 60:
options = self._ffi.cast(
'uint8_t *',
iph6) + self._ffi.sizeof('struct nfstream_ipv6hdr')
proto = options[0]
l4_packet_len = ntohs(iph6.ip6_hdr.ip6_un1_plen)
nfstream_hash += (
iph6.ip6_src.u6_addr.u6_addr32[2] +
iph6.ip6_src.u6_addr.u6_addr32[3]) + (
iph6.ip6_dst.u6_addr.u6_addr32[2] +
iph6.ip6_dst.u6_addr.u6_addr32[3]) + proto + vlan_id
if version == 4:
if ipsize < 20:
return None
if ((iph.ihl * 4) > ipsize) or (ipsize < ntohs(iph.tot_len)):
return None
l4_offset = iph.ihl * 4
l3 = self._ffi.cast('uint8_t *', iph)
else:
l4_offset = self._ffi.sizeof('struct nfstream_ipv6hdr')
l3 = self._ffi.cast('uint8_t *', iph6)
l4 = self._ffi.cast('uint8_t *', l3) + l4_offset
syn, cwr, ece, urg, ack, psh, rst, fin = 0, 0, 0, 0, 0, 0, 0, 0
if (proto == 6
) and l4_packet_len >= self._ffi.sizeof('struct nfstream_tcphdr'):
tcph = self._ffi.cast('struct nfstream_tcphdr *', l4)
sport = int(ntohs(tcph.source))
dport = int(ntohs(tcph.dest))
syn = int(tcph.syn)
cwr = int(tcph.cwr)
ece = int(tcph.ece)
urg = int(tcph.urg)
ack = int(tcph.ack)
psh = int(tcph.psh)
rst = int(tcph.rst)
fin = int(tcph.fin)
elif (proto == 17) and l4_packet_len >= self._ffi.sizeof(
'struct nfstream_udphdr'):
udph = self._ffi.cast('struct nfstream_udphdr *', l4)
sport = int(ntohs(udph.source))
dport = int(ntohs(udph.dest))
else:
sport = 0
dport = 0
nfstream_hash += sport + dport
if version == 4:
return NFPacket(time=int(time),
capture_length=header.caplen,
length=header.len,
nfhash=nfstream_hash,
ip_src=src_addr,
ip_dst=dst_addr,
src_port=sport,
dst_port=dport,
protocol=proto,
vlan_id=vlan_id,
version=version,
tcpflags=tcpflags(syn=syn,
cwr=cwr,
ece=ece,
urg=urg,
ack=ack,
psh=psh,
rst=rst,
fin=fin),
raw=bytes(xffi.buffer(iph, ipsize)),
root_idx=nfstream_hash % nroots)
else:
return NFPacket(time=int(time),
capture_length=header.caplen,
length=header.len,
nfhash=nfstream_hash,
ip_src=src_addr,
ip_dst=dst_addr,
src_port=sport,
dst_port=dport,
protocol=proto,
vlan_id=vlan_id,
version=version,
tcpflags=tcpflags(syn=syn,
cwr=cwr,
ece=ece,
urg=urg,
ack=ack,
psh=psh,
rst=rst,
fin=fin),
raw=bytes(xffi.buffer(iph6,
header.len - ip_offset)),
root_idx=nfstream_hash % nroots)
def _recv_packet(self, xdev, nroots=1):
phdr = self._ffi.new("struct pcap_pkthdr **")
pdata = self._ffi.new("unsigned char **")
rv = self._libpcap.pcap_next_ex(xdev, phdr, pdata)
if rv == 1:
return self._process_packet(xdev, phdr[0], pdata[0], nroots)
elif rv == 0:
# timeout; nothing to return
return 0
elif rv == -1:
# error on receive; raise an exception
s = self._ffi.string(self._libpcap.pcap_geterr(xdev))
raise PcapException("Error receiving packet: {}".format(s))
elif rv == -2:
# reading from savefile, but none left
return -2
#!/usr/bin/env python
from cffi import FFI
import numpy as np
# Array data for a 4x3 RGB (black) image.
dtype = [("r", np.uint8), ("g", np.uint8), ("b", np.uint8)]
array = np.zeros((3,4), dtype=dtype)
# declare data & function for cffi
ffi = FFI()
data = ffi.cast("unsigned char*", array.__array_interface__["data"][0])
length = ffi.cast("unsigned int", array.size)
ffi.cdef("void greenify(unsigned char* data, unsigned int length);")
# open the C lib and call a C function
libgreen = ffi.dlopen("./libgreen.so")
libgreen.greenify(data, length)
# the array has been modified.
print array
class LibWwqParseCFFI(LibWwqLyParseBase):
def __init__(self):
super(LibWwqParseCFFI, self).__init__()
from cffi import FFI
self.ffi = FFI()
self.lib = self.ffi.dlopen(self.lib_path)
self.ffi.cdef("""
char * get_uuid();
char * get_name();
int parse(char * c,int length,char **result,int *result_length);
int free_str(char * c);
void* atomic_int64_init();
int atomic_int64_destroy(void* ptr);
int64_t atomic_int64_get(void* ptr);
int64_t atomic_int64_set(void* ptr, int64_t val);
int64_t atomic_int64_add(void* ptr, int64_t val);
int64_t atomic_int64_sub(void* ptr, int64_t val);
int64_t atomic_int64_and(void* ptr, int64_t val);
int64_t atomic_int64_or(void* ptr, int64_t val);
int64_t atomic_int64_xor(void* ptr, int64_t val);
typedef union epoll_data {
void* ptr;
int fd;
uint32_t u32;
uint64_t u64;
uintptr_t sock; /* Windows specific */
void* hnd; /* Windows specific */
} epoll_data_t;
typedef struct {
uint32_t events; /* Epoll events and flags */
epoll_data_t data; /* User data variable */
} epoll_event ;
void* epoll_create(int size);
void* epoll_create1(int flags);
int epoll_close(void* ephnd);
int epoll_ctl(void* ephnd, int op, uintptr_t sock, epoll_event* event);
int epoll_wait(void* ephnd, epoll_event* events, int maxevents, int timeout);
""")
self.lib.__class__.__repr__ = lambda s: "<%s object at 0x%016X>" % (s.__class__.__name__, id(s))
logging.debug("successful load lib %s" % self.lib)
weakref.finalize(self,
lambda: logging.debug("%s released" % self.lib) if self.ffi.dlclose(self.lib) or 1 else None)
def get_uuid(self) -> bytes:
return self.ffi.string(self.lib.get_uuid())
def get_name(self) -> bytes:
return self.ffi.string(self.lib.get_name())
def lib_parse(self, byte_str: bytes) -> bytes:
length = self.ffi.cast("int", len(byte_str))
result_length = self.ffi.new("int *")
result_p = self.ffi.new("char **")
# p = self.ffi.new("char []", byte_str)
p = self.ffi.from_buffer(byte_str)
self.lib.parse(p, length, result_p, result_length)
result = self.ffi.unpack(result_p[0], result_length[0])
self.lib.free_str(result_p[0])
return result
{
int id;
double* xk;
double h;
double theta;
double gamma;
double g;
double kappa;
unsigned int f_eval;
unsigned int nabla_eval;
} data;
''')
data_struct = ffi.new('data*')
data_struct.id = -1 # to avoid freeing the data in the destructor
data_struct.xk = ffi.cast('double *', xk.ctypes.data)
data_struct.h = h
data_struct.theta = theta
data_struct.gamma = gamma
data_struct.g = g
data_struct.kappa = kappa
D = ffi.dlopen(SN._numerics.__file__)
D.set_cstruct(mcp.get_env_as_long(), ffi.cast('void*', data_struct))
mcp.set_compute_F_and_nabla_F_as_C_functions('ZhuravlevIvanov.so', 'compute_Fmcp', 'compute_nabla_Fmcp')
def test_explicitly_defined_char16_t(self):
ffi = FFI()
ffi.cdef("typedef uint16_t char16_t;")
x = ffi.cast("char16_t", 1234)
assert ffi.typeof(x) is ffi.typeof("uint16_t")
class H264Decoder:
def __init_ffi(self):
self.ffi = FFI()
self.ffi.cdef('''
// AVCODEC
enum PixelFormat { PIX_FMT_YUV420P, PIX_FMT_RGB24, ... };
void avcodec_register_all(void);
struct AVPacket { ...; uint8_t *data; int size; ...; };
void av_init_packet(struct AVPacket *pkt);
enum AVCodecID { AV_CODEC_ID_H264, ... };
struct AVCodec *avcodec_find_decoder(enum AVCodecID id);
struct AVCodecContext *avcodec_alloc_context3(struct AVCodec *codec);
int avcodec_open2(struct AVCodecContext *avctx, struct AVCodec *codec,
struct AVDictionary **options);
struct AVFrame { uint8_t *data[8]; int linesize[8]; ...; int key_frame; ...; };
struct AVFrame *avcodec_alloc_frame(void);
int avcodec_decode_video2(struct AVCodecContext *avctx, struct AVFrame *picture,
int *got_picture_ptr, struct AVPacket *avpkt);
int avcodec_close(struct AVCodecContext *avctx);
void av_free(void *ptr);
int avpicture_get_size(enum PixelFormat pix_fmt, int width, int height);
int avpicture_fill(struct AVPicture *picture, uint8_t *ptr,
int pix_fmt, int width, int height);
// SWSCALE
#define SWS_BILINEAR ...
#define SWS_FAST_BILINEAR ...
struct SwsContext *sws_getContext(int srcW, int srcH, enum PixelFormat srcFormat,
int dstW, int dstH, enum PixelFormat dstFormat,
int flags, struct SwsFilter *srcFilter,
struct SwsFilter *dstFilter, const double *param);
int sws_scale(struct SwsContext *c, const uint8_t *const srcSlice[],
const int srcStride[], int srcSliceY,
int srcSliceH, uint8_t *const dst[],
const int dstStride[]);
void sws_freeContext(struct SwsContext *c);
''')
self.ns = self.ffi.verify(source='''
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>
''',
libraries=['avcodec', 'swscale'])
def __init_avcodec(self):
self.ns.avcodec_register_all()
self.av_packet = self.ffi.new('struct AVPacket *')
self.ns.av_init_packet(self.av_packet)
self.codec = self.ns.avcodec_find_decoder(self.ns.AV_CODEC_ID_H264)
if not self.codec:
raise Exception('avcodec_alloc_context3')
self.context = self.ns.avcodec_alloc_context3(self.codec)
if not self.context:
raise Exception('avcodec_alloc_context3')
if self.ns.avcodec_open2(self.context, self.codec, self.ffi.NULL) < 0:
raise Exception('avcodec_open2')
self.frame = self.ns.avcodec_alloc_frame()
if not self.frame:
raise Exception('avcodec_alloc_frame')
self.got_frame = self.ffi.new('int *')
self.out_frame = self.ns.avcodec_alloc_frame()
def __init__(self):
self.out_buffer, self.sws_context = None, None
self.__init_ffi()
self.__init_avcodec()
self.update_dimensions()
def close(self):
self.ns.sws_freeContext(self.sws_context)
self.ns.av_free(self.out_frame)
self.ns.avcodec_close(self.context)
self.ns.av_free(self.context)
self.ns.av_free(self.frame)
def update_dimensions(self):
if self.sws_context is not None:
self.ns.sws_freeContext(self.sws_context)
self.sws_context = self.ns.sws_getContext(
constants.WII_VIDEO_WIDTH, constants.WII_VIDEO_HEIGHT,
self.ns.PIX_FMT_YUV420P, constants.WII_VIDEO_WIDTH,
constants.WII_VIDEO_HEIGHT, self.ns.PIX_FMT_RGB24,
self.ns.SWS_FAST_BILINEAR, self.ffi.NULL, self.ffi.NULL,
self.ffi.NULL)
bytes_req = self.ns.avpicture_get_size(self.ns.PIX_FMT_RGB24,
constants.WII_VIDEO_WIDTH,
constants.WII_VIDEO_HEIGHT)
self.out_buffer = self.ffi.new('uint8_t [%i]' % bytes_req)
self.ns.avpicture_fill(
self.ffi.cast('struct AVPicture *', self.out_frame),
self.out_buffer, self.ns.PIX_FMT_RGB24, constants.WII_VIDEO_WIDTH,
constants.WII_VIDEO_HEIGHT)
def get_image_buffer(self, encoded_nalu):
in_data = self.ffi.new('uint8_t []', encoded_nalu)
self.av_packet.data = in_data
self.av_packet.size = len(encoded_nalu)
length = self.ns.avcodec_decode_video2(self.context, self.frame,
self.got_frame, self.av_packet)
if length < 0:
raise Exception('avcodec_decode_video2')
elif length != self.av_packet.size:
raise Exception('expected to decode a single complete frame')
elif self.got_frame[0]:
# print 'keyframe:', s.frame.key_frame
# convert from YUV to RGB
self.ns.sws_scale(self.sws_context, self.frame.data,
self.frame.linesize, 0,
constants.WII_VIDEO_HEIGHT, self.out_frame.data,
self.out_frame.linesize)
image_buffer = \
self.ffi.buffer(self.out_frame.data[0], self.out_frame.linesize[0] * constants.WII_VIDEO_HEIGHT)
return image_buffer
import time
import numpy as np
from cffi import FFI
from scipy.spatial.distance import cdist
from _cityblock.lib import cbdm
nsamples = 12000
nfeat = 50
x = np.random.random([nsamples, nfeat])
r = np.empty((nsamples, nsamples))
ffi = FFI()
x_c = ffi.cast('double *', x.ctypes.data)
r_c = ffi.cast('double *', r.ctypes.data)
start = time.time()
cbdm(x_c, x_c, r_c, nsamples, nfeat)
print(f'C : {time.time() - start:.2f} seconds')
start = time.time()
r_cdist = cdist(x, x, 'cityblock')
print(f'cdist : {time.time() - start:.2f} seconds')
# check the result by comparing to cdist
print(f'\ndiff : {(r - r_cdist).max():.2e}')
# reply = C.uct_notify_play(engine, board, move, null_cstr)
# ret = C.board_play(board, move)
# C.board_print_stderr(board)
move = ffi.new("struct move*")
C = None
try:
import os
print os.path.exists(os.path.join(os.getcwd(), "libpachi.so"))
C = ffi.dlopen("libpachi.so") # loads the entire C namespace
except Exception as e:
print e
print("trying osx")
if C == None:
C = ffi.dlopen("libpachi.dylib") # loads the entire C namespace
null_cstr = ffi.cast("char*", 0)
board = C.board_init(null_cstr)
size = 9
C.board_resize(board, size)
C.board_clear(board)
engine = C.engine_uct_init(null_cstr, board)
t_black = C.time_info_init()
t_white = C.time_info_init()
class myHandler(BaseHTTPRequestHandler):
def do_GET(self):
self.do_POST()
def do_POST(self):
self.log_message("do_POST %s" % self.path)
self.send_response(200)
self.send_header('Content-type','application/json')
class cv2pynq():
MAX_WIDTH = 1920
MAX_HEIGHT = 1080
def __init__(self, load_overlay=True):
self.bitstream_name = None
self.bitstream_name = "cv2pynq03.bit"
self.bitstream_path = os.path.join(CV2PYNQ_BIT_DIR,
self.bitstream_name)
self.ol = Overlay(self.bitstream_path)
self.ol.download()
self.ol.reset()
self.xlnk = Xlnk()
self.partitions = 10 #split the cma into partitions for pipelined transfer
self.cmaPartitionLen = self.MAX_HEIGHT * self.MAX_WIDTH / self.partitions
self.listOfcma = [
self.xlnk.cma_array(shape=(int(self.MAX_HEIGHT / self.partitions),
self.MAX_WIDTH),
dtype=np.uint8) for i in range(self.partitions)
]
self.img_filters = self.ol.image_filters
self.dmaOut = self.img_filters.axi_dma_0.sendchannel
self.dmaIn = self.img_filters.axi_dma_0.recvchannel
self.dmaOut.stop()
self.dmaIn.stop()
self.dmaIn.start()
self.dmaOut.start()
self.filter2DType = -1 # filter types: SobelX=0, SobelY=1, ScharrX=2, ScharrY=3, Laplacian1=4, Laplacian3=5
self.filter2D_5Type = -1 # filter types: SobelX=0, SobelY=1, Laplacian5=4
self.filter2DfType = -1 # filter types: blur=0, GaussianBlur=1
self.ffi = FFI()
self.f2D = self.img_filters.filter2D_hls_0
self.f2D.reset()
self.f2D_5 = self.img_filters.filter2D_hls_5_0
self.f2D_5.reset()
self.f2D_f = self.img_filters.filter2D_f_0
self.f2D_f.reset()
self.erodeIP = self.img_filters.erode_hls_0
self.erodeIP.reset()
self.dilateIP = self.img_filters.dilate_hls_0
self.dilateIP.reset()
self.cmaBuffer_0 = self.xlnk.cma_array(shape=(self.MAX_HEIGHT,
self.MAX_WIDTH),
dtype=np.uint8)
self.cmaBuffer0 = self.cmaBuffer_0.view(self.ContiguousArrayCv2pynq)
self.cmaBuffer0.init(self.cmaBuffer_0)
self.cmaBuffer_1 = self.xlnk.cma_array(shape=(self.MAX_HEIGHT,
self.MAX_WIDTH),
dtype=np.uint8)
self.cmaBuffer1 = self.cmaBuffer_1.view(self.ContiguousArrayCv2pynq)
self.cmaBuffer1.init(self.cmaBuffer_1)
self.cmaBuffer_2 = self.xlnk.cma_array(
shape=(self.MAX_HEIGHT * 4, self.MAX_WIDTH),
dtype=np.uint8) # *4 for CornerHarris return
self.cmaBuffer2 = self.cmaBuffer_2.view(self.ContiguousArrayCv2pynq)
self.cmaBuffer2.init(self.cmaBuffer_2)
self.CannyIP = self.img_filters.canny_edge_0
self.CannyIP.reset()
#self.cornerHarrisIP = self.img_filters.CornerHarris_hls_0
#self.cornerHarrisIP.reset()
def close(self):
#self.dmaOut.stop()
#self.dmaIn.stop()
self.cmaBuffer_0.close()
self.cmaBuffer_1.close()
self.cmaBuffer_2.close()
for cma in self.listOfcma:
cma.close()
def Sobel(self, src, ddepth, dx, dy, dst, ksize):
if (ksize == 3):
self.f2D.rows = src.shape[0]
self.f2D.columns = src.shape[1]
self.f2D.channels = 1
if (dx == 1) and (dy == 0):
if self.filter2DType != 0:
self.filter2DType = 0
self.f2D.r1 = 0x000100ff #[-1 0 1]
self.f2D.r2 = 0x000200fe #[-2 0 2]
self.f2D.r3 = 0x000100ff #[-1 0 1]
elif (dx == 0) and (dy == 1):
if self.filter2DType != 1:
self.filter2DType = 1
self.f2D.r1 = 0x00fffeff #[-1 -2 -1]
self.f2D.r2 = 0x00000000 #[ 0 0 0]
self.f2D.r3 = 0x00010201 #[ 1 2 1]
else:
raise RuntimeError("Incorrect dx dy configuration")
self.img_filters.select_filter(1)
self.f2D.start()
return self.filter2D(src, dst)
else: #ksize == 5
self.f2D_5.rows = src.shape[0]
self.f2D_5.columns = src.shape[1]
if (dx == 1) and (dy == 0):
if self.filter2D_5Type != 0:
self.filter2D_5Type = 0
self.f2D_5.par_V = bytes([ \
#-1, -2, 0, 2, 1,
0xff, 0xfe, 0x00, 0x02, 0x01, \
#-4, -8, 0, 8, 4,
0xfc, 0xf8, 0x00, 0x08, 0x04, \
#-6, -12, 0, 12, 6,
0xfa, 0xf4, 0x00, 0x0c, 0x06, \
#-4, -8, 0, 8, 4,
0xfc, 0xf8, 0x00, 0x08, 0x04, \
#-1, -2, 0, 2, 1,
0xff, 0xfe, 0x00, 0x02, 0x01, \
0,0,0]) #fill up to allign with 4
elif (dx == 0) and (dy == 1):
if self.filter2D_5Type != 1:
self.filter2D_5Type = 1
self.f2D_5.par_V = bytes([ \
#-1, -4, -6, -4, -1,
0xff, 0xfc, 0xfa, 0xfc, 0xff, \
#-2, -8, -12, -8, -2,
0xfe, 0xf8, 0xf4, 0xf8, 0xfe, \
# 0, 0, 0, 0, 0,
0x00, 0x00, 0x00, 0x00, 0x00, \
# 2, 8, 12, 8, 2,
0x02, 0x08, 0x0c, 0x08, 0x02, \
# 1, 4, 6, 4, 1,
0x01, 0x04, 0x06, 0x04, 0x01, \
0,0,0]) #fill up to allign with 4
else:
raise RuntimeError("Incorrect dx dy configuration")
self.img_filters.select_filter(5)
self.f2D_5.start()
return self.filter2D(src, dst)
def Scharr(self, src, ddepth, dx, dy, dst):
self.f2D.rows = src.shape[0]
self.f2D.columns = src.shape[1]
self.f2D.channels = 1
if (dx == 1) and (dy == 0):
if self.filter2DType != 2:
self.filter2DType = 2
self.f2D.r1 = 0x000300fd #[-3 0 3]
self.f2D.r2 = 0x000a00f6 #[-10 0 10]
self.f2D.r3 = 0x000300fd #[-3 0 3]
elif (dx == 0) and (dy == 1):
if self.filter2DType != 3:
self.filter2DType = 3
self.f2D.r1 = 0x00fdf6fd #[-3 -10 -3]
self.f2D.r2 = 0x00000000 #[ 0 0 0]
self.f2D.r3 = 0x00030a03 #[ 3 10 3]
else:
raise RuntimeError("Incorrect dx dy configuration")
self.img_filters.select_filter(1)
self.f2D.start()
return self.filter2D(src, dst)
def Laplacian(self, src, ddepth, dst, ksize):
if ksize == 5:
self.f2D_5.rows = src.shape[0]
self.f2D_5.columns = src.shape[1]
if self.filter2D_5Type != 4:
self.filter2D_5Type = 4 # "Laplacian_5"
self.f2D_5.par_V = bytes([ \
#2, 4, 4, 4, 2,
0x02, 0x04, 0x04, 0x04, 0x02, \
#4, 0, -8, 0, 4,
0x04, 0x00, 0xf8, 0x00, 0x04, \
#4, -8, -24, -8, 4,
0x04, 0xf8, 0xe8, 0xf8, 0x04, \
#4, 0, -8, 0, 4,
0x04, 0x00, 0xf8, 0x00, 0x04, \
#2, 4, 4, 4, 2,
0x02, 0x04, 0x04, 0x04, 0x02, \
0,0,0]) #fill up to allign with 4
self.img_filters.select_filter(5)
self.f2D_5.start()
return self.filter2D(src, dst)
else: #ksize 1 or 3
self.f2D.rows = src.shape[0]
self.f2D.columns = src.shape[1]
self.f2D.channels = 1
if ksize == 1:
if (self.filter2DType != 4):
self.filter2DType = 4 # "Laplacian_1"
self.f2D.r1 = 0x00000100 #[ 0 1 0]
self.f2D.r2 = 0x0001fc01 #[ 1 -4 1]
self.f2D.r3 = 0x00000100 #[ 0 1 0]
elif ksize == 3:
if (self.filter2DType != 5):
self.filter2DType = 5 # "Laplacian_3"
self.f2D.r1 = 0x00020002 #[ 2 0 2]
self.f2D.r2 = 0x0000f800 #[ 0 -8 0]
self.f2D.r3 = 0x00020002 #[ 2 0 2]
self.img_filters.select_filter(1)
self.f2D.start()
return self.filter2D(src, dst)
def blur(self, src, ksize, dst):
self.f2D_f.rows = src.shape[0]
self.f2D_f.columns = src.shape[1]
if (self.filter2DfType != 0):
self.filter2DfType = 0 #blur
mean = self.floatToFixed(1 / 9, cv2pynqDriverFilter2D_f.K_FP_W,
cv2pynqDriverFilter2D_f.K_FP_F)
self.f2D_f.r11 = mean
self.f2D_f.r12 = mean
self.f2D_f.r13 = mean
self.f2D_f.r21 = mean
self.f2D_f.r22 = mean
self.f2D_f.r23 = mean
self.f2D_f.r31 = mean
self.f2D_f.r32 = mean
self.f2D_f.r33 = mean
self.img_filters.select_filter(2)
self.f2D_f.start()
return self.filter2D(src, dst)
def GaussianBlur(self, src, ksize, sigmaX, sigmaY, dst):
self.f2D_f.rows = src.shape[0]
self.f2D_f.columns = src.shape[1]
if (self.filter2DfType != 1):
self.filter2DfType = 1 #GaussianBlur
if (sigmaX <= 0):
sigmaX = 0.3 * ((ksize[0] - 1) * 0.5 - 1) + 0.8
if (sigmaY <= 0):
sigmaY = sigmaX
kX = cv2.getGaussianKernel(3, sigmaX, ktype=cv2.CV_32F) #kernel X
kY = cv2.getGaussianKernel(3, sigmaY, ktype=cv2.CV_32F) #kernel Y
self.f2D_f.r11 = self.floatToFixed(kY[0] * kX[0],
cv2pynqDriverFilter2D_f.K_FP_W,
cv2pynqDriverFilter2D_f.K_FP_F)
self.f2D_f.r12 = self.floatToFixed(kY[0] * kX[1],
cv2pynqDriverFilter2D_f.K_FP_W,
cv2pynqDriverFilter2D_f.K_FP_F)
self.f2D_f.r13 = self.floatToFixed(kY[0] * kX[2],
cv2pynqDriverFilter2D_f.K_FP_W,
cv2pynqDriverFilter2D_f.K_FP_F)
self.f2D_f.r21 = self.floatToFixed(kY[1] * kX[0],
cv2pynqDriverFilter2D_f.K_FP_W,
cv2pynqDriverFilter2D_f.K_FP_F)
self.f2D_f.r22 = self.floatToFixed(kY[1] * kX[1],
cv2pynqDriverFilter2D_f.K_FP_W,
cv2pynqDriverFilter2D_f.K_FP_F)
self.f2D_f.r23 = self.floatToFixed(kY[1] * kX[2],
cv2pynqDriverFilter2D_f.K_FP_W,
cv2pynqDriverFilter2D_f.K_FP_F)
self.f2D_f.r31 = self.floatToFixed(kY[2] * kX[0],
cv2pynqDriverFilter2D_f.K_FP_W,
cv2pynqDriverFilter2D_f.K_FP_F)
self.f2D_f.r32 = self.floatToFixed(kY[2] * kX[1],
cv2pynqDriverFilter2D_f.K_FP_W,
cv2pynqDriverFilter2D_f.K_FP_F)
self.f2D_f.r33 = self.floatToFixed(kY[2] * kX[2],
cv2pynqDriverFilter2D_f.K_FP_W,
cv2pynqDriverFilter2D_f.K_FP_F)
self.img_filters.select_filter(2)
self.f2D_f.start()
return self.filter2D(src, dst)
def erode(self, src, kernel, dst, iterations, mode):
self.img_filters.select_filter(3)
return self.erodeDilateKernel(src, kernel, dst, iterations, mode,
self.erodeIP)
def dilate(self, src, kernel, dst, iterations, mode):
self.img_filters.select_filter(4)
return self.erodeDilateKernel(src, kernel, dst, iterations, mode,
self.dilateIP)
def Canny(self, src, threshold1, threshold2, dst):
self.img_filters.select_filter(0)
self.CannyIP.rows = src.shape[0]
self.CannyIP.columns = src.shape[1]
self.CannyIP.threshold1 = threshold1
self.CannyIP.threshold2 = threshold2
self.CannyIP.start()
if hasattr(src, 'physical_address') and hasattr(
dst, 'physical_address'):
self.dmaIn.transfer(dst)
self.dmaOut.transfer(src)
self.dmaIn.wait()
return dst
self.cmaBuffer1.nbytes = src.nbytes
self.dmaIn.transfer(self.cmaBuffer1)
if hasattr(src, 'physical_address'):
self.dmaOut.transfer(src)
else:
self.cmaBuffer0.nbytes = src.nbytes
self.copyNto(self.cmaBuffer0, src, src.nbytes)
self.dmaOut.transfer(self.cmaBuffer0)
self.dmaIn.wait()
ret = np.ndarray(src.shape, src.dtype)
self.copyNto(ret, self.cmaBuffer1, ret.nbytes)
return ret
def filter2D(self, src, dst):
if dst is None:
self.cmaBuffer1.nbytes = src.nbytes
elif hasattr(src, 'physical_address') and hasattr(
dst, 'physical_address'):
self.dmaIn.transfer(dst)
self.dmaOut.transfer(src)
self.dmaIn.wait()
return dst
if hasattr(src, 'physical_address'):
self.dmaIn.transfer(self.cmaBuffer1)
self.dmaOut.transfer(src)
self.dmaIn.wait()
else: #pipeline the copy to contiguous memory and filter calculation in hardware
if src.nbytes < 184800: #440x420
self.partitions = 1
elif src.nbytes < 180000: #600x300
self.partitions = 2
elif src.nbytes < 231200: #680x340
self.partitions = 4
else:
self.partitions = 8
self.cmaBuffer1.nbytes = src.nbytes
self.dmaIn.transfer(self.cmaBuffer1)
chunks_len = int(src.nbytes / (self.partitions))
self.cmaBuffer0.nbytes = chunks_len
self.cmaBuffer2.nbytes = chunks_len
#self.copyNto(src,self.cmaBuffer0,chunks_len)
self.copyNto(self.cmaBuffer0, src, chunks_len)
for i in range(1, self.partitions):
if i % 2 == 1:
while not self.dmaOut.idle and not self.dmaOut._first_transfer:
pass
self.dmaOut.transfer(self.cmaBuffer0)
#self.copyNtoOff(src ,self.cmaBuffer2,chunks_len, i*chunks_len, 0)
self.copyNtoOff(self.cmaBuffer2, src, chunks_len, 0,
i * chunks_len)
else:
while not self.dmaOut.idle and not self.dmaOut._first_transfer:
pass
self.dmaOut.transfer(self.cmaBuffer2)
#self.copyNtoOff(src ,self.cmaBuffer0,chunks_len, i*chunks_len, 0)
self.copyNtoOff(self.cmaBuffer0, src, chunks_len, 0,
i * chunks_len)
while not self.dmaOut.idle and not self.dmaOut._first_transfer:
pass
self.dmaOut.transfer(self.cmaBuffer2)
rest = src.nbytes % self.partitions
if rest > 0: #cleanup any remaining data and send it to HW
#self.copyNtoOff(src ,self.cmaBuffer0,chunks_len, self.partitions*chunks_len, 0)
self.copyNtoOff(self.cmaBuffer0, src, chunks_len, 0,
self.partitions * chunks_len)
while not self.dmaOut.idle and not self.dmaOut._first_transfer:
pass
self.dmaOut.transfer(self.cmaBuffer0)
rest -= chunks_len
self.dmaIn.wait()
ret = np.ndarray(src.shape, src.dtype)
self.copyNto(ret, self.cmaBuffer1, ret.nbytes)
return ret
def floatToFixed(self, f, total_bits, fract_bits):
"""convert float f to a signed fixed point with #total_bits and #frac_bits after the point"""
fix = int((abs(f) * (1 << fract_bits)))
if (f < 0):
fix += 1 << total_bits - 1
return fix
def erodeDilateKernel(self, src, kernel, dst, iterations, mode, filter):
filter.mode = mode
filter.rows = src.shape[0]
filter.columns = src.shape[1]
if hasattr(src, 'physical_address') and hasattr(
dst, 'physical_address'):
filter.start()
if iterations > 1:
self.dmaIn.transfer(self.cmaBuffer1)
else:
self.dmaIn.transfer(dst)
self.dmaOut.transfer(src)
self.dmaIn.wait()
self.cmaBuffer2.nbytes = src.nbytes #buffer = self.xlnk.cma_array(src.shape, dtype=np.uint8)
for i in range(2, iterations + 1):
filter.start()
if i % 2 == 0:
self.dmaIn.transfer(self.cmaBuffer2)
if i != iterations: #avoid copy after last iteration
self.dmaOut.transfer(self.cmaBuffer1)
else:
self.dmaOut.transfer(dst)
else:
self.dmaIn.transfer(self.cmaBuffer1)
if i != iterations:
self.dmaOut.transfer(self.cmaBuffer2)
else:
self.dmaOut.transfer(dst)
self.dmaIn.wait()
return dst
self.cmaBuffer0.nbytes = src.nbytes
self.cmaBuffer1.nbytes = src.nbytes
filter.start()
self.dmaIn.transfer(self.cmaBuffer1)
if hasattr(src, 'physical_address'):
self.dmaOut.transfer(src)
else:
self.copyNto(self.cmaBuffer0, src,
src.nbytes) #np.copyto(srcBuffer,src)
self.dmaOut.transfer(self.cmaBuffer0)
self.dmaIn.wait()
self.cmaBuffer2.nbytes = src.nbytes #buffer = self.xlnk.cma_array(src.shape, dtype=np.uint8)
for i in range(2, iterations + 1):
filter.start()
if i % 2 == 0:
self.dmaIn.transfer(self.cmaBuffer2)
self.dmaOut.transfer(self.cmaBuffer1)
else:
self.dmaIn.transfer(self.cmaBuffer1)
self.dmaOut.transfer(self.cmaBuffer2)
self.dmaIn.wait()
ret = np.ndarray(src.shape, src.dtype)
if iterations % 2 == 1:
self.copyNto(ret, self.cmaBuffer1, ret.nbytes)
else:
self.copyNto(ret, self.cmaBuffer2, ret.nbytes)
return ret
'''def cornerHarris(self, src, k, dst):
self.img_filters.select_filter(5)
self.cornerHarrisIP.rows = src.shape[0]
self.cornerHarrisIP.columns = src.shape[1]
self.cornerHarrisIP.start()
if hasattr(src, 'physical_address') and hasattr(dst, 'physical_address') and (dst.nbytes == src.nbytes*4):
self.dmaIn.transfer(dst)
self.dmaOut.transfer(src)
self.dmaIn.wait()
return dst
self.cmaBuffer2.nbytes = src.nbytes*4
self.dmaIn.transfer(self.cmaBuffer2)
if hasattr(src, 'physical_address') :
self.dmaOut.transfer(src)
else:
self.cmaBuffer0.nbytes = src.nbytes
self.copyNto(self.cmaBuffer0,src,src.nbytes)
self.dmaOut.transfer(self.cmaBuffer0)
self.dmaIn.wait()
ret = np.ndarray(src.shape,np.float32)
self.copyNto(ret,self.cmaBuffer2,ret.nbytes)
return ret'''
def copyNto(self, dst, src, N):
dstPtr = self.ffi.cast("uint8_t *", self.ffi.from_buffer(dst))
srcPtr = self.ffi.cast("uint8_t *", self.ffi.from_buffer(src))
self.ffi.memmove(dstPtr, srcPtr, N)
def copyNtoOff(self, dst, src, N, dstOffset, srcOffset):
dstPtr = self.ffi.cast("uint8_t *", self.ffi.from_buffer(dst))
srcPtr = self.ffi.cast("uint8_t *", self.ffi.from_buffer(src))
dstPtr += dstOffset
srcPtr += srcOffset
self.ffi.memmove(dstPtr, srcPtr, N)
class ContiguousArrayCv2pynq(ContiguousArray):
def init(self, cmaArray):
self._nbytes = cmaArray.nbytes
self.physical_address = cmaArray.physical_address
self.cacheable = cmaArray.cacheable
# overwrite access to nbytes with own function
@property
def nbytes(self):
return self._nbytes
@nbytes.setter
def nbytes(self, value):
self._nbytes = value
def test_vi_C_interface():
try:
from cffi import FFI
cffi_is_present = True
except:
cffi_is_present = False
return
if cffi_is_present:
h = 1e-5
T = 1.0
t = 0.0
theta = 1.0
gamma = 1.0
g = 9.81
kappa = 0.4
xk = np.array((1., 10.))
ffi = FFI()
ffi.cdef('void set_cstruct(uintptr_t p_env, void* p_struct);')
ffi.cdef('''typedef struct
{
int id;
double* xk;
double h;
double theta;
double gamma;
double g;
double kappa;
unsigned int f_eval;
unsigned int nabla_eval;
} data;
''')
data_struct = ffi.new('data*')
data_struct.id = -1 # to avoid freeing the data in the destructor
data_struct.xk = ffi.cast('double *', xk.ctypes.data)
data_struct.h = h
data_struct.theta = theta
data_struct.gamma = gamma
data_struct.g = g
data_struct.kappa = kappa
vi = SN.VI(2)
import siconos
D = ffi.dlopen(siconos.__path__[0] + '/_numerics.so')
D.set_cstruct(vi.get_env_as_long(), ffi.cast('void*', data_struct))
vi.set_compute_F_and_nabla_F_as_C_functions('ZhuravlevIvanov.so',
'compute_F',
'compute_nabla_F')
lambda_ = np.zeros((2, ))
xkp1 = np.zeros((2, ))
SO = SN.SolverOptions(vi, SN.SICONOS_VI_BOX_QI)
lb = np.array((-1.0, -1.0))
ub = np.array((1.0, 1.0))
vi.set_box_constraints(lb, ub)
N = int(T / h + 10)
print(N)
SO.dparam[0] = 1e-24
SO.iparam[0] = 100
SO.iparam[2] = 1
SO.iparam[3] = 0
SO.iparam[4] = 5
signs = np.empty((N, 2))
sol = np.empty((N, 2))
sol[0, :] = xk
k = 0
#SN.numerics_set_verbose(3)
while t <= T:
k += 1
info = SN.variationalInequality_box_newton_QiLSA(
vi, lambda_, xkp1, SO)
#print('iter {:} ; solver iter = {:} ; prec = {:}'.format(k, SO.iparam[1], SO.dparam[1]))
if info > 0:
print(lambda_)
# vi_function(2, signs[k-1, :], xkp1)
lambda_[0] = -np.sign(xkp1[0])
lambda_[1] = -np.sign(xkp1[1])
if np.abs(xk[0]) < 1e-10:
lambda_[0] = 0.01
if np.abs(xk[1]) < 1e-10:
lambda_[1] = 0.01
print('ok lambda')
print(lambda_)
info = SN.variationalInequality_box_newton_QiLSA(
vi, lambda_, xkp1, SO)
print('iter {:} ; solver iter = {:} ; prec = {:}'.format(
k, SO.iparam[1], SO.dparam[1]))
if info > 0:
print('VI solver failed ! info = {:}'.format(info))
print(xk)
print(lambda_)
print(xkp1)
kaboom()
# else:
# print('iter {:} ; solver iter = {:} ; prec = {:}'.format(k, SO.iparam[1], SO.dparam[1]))
# vi_function(2, lambda_, xkp1)
sol[k, 0:2] = xkp1
np.copyto(xk, xkp1, casting='no')
signs[k, 0:2] = lambda_
t = k * h
def runKernel(opt):
ffi = FFI() # create the FFI obj
boHandle = xclAllocBO(opt.handle, opt.DATA_SIZE, xclBOKind.XCL_BO_DEVICE_RAM, opt.first_mem)
bo1 = xclMapBO(opt.handle, boHandle, True)
read_fp = ffi.cast("FILE *", bo1)
if xclSyncBO(opt.handle, boHandle, xclBOSyncDirection.XCL_BO_SYNC_BO_TO_DEVICE, opt.DATA_SIZE, 0):
return 1
p = xclBOProperties()
bodevAddr = p.paddr if not (xclGetBOProperties(opt.handle, boHandle, p)) else -1
if bodevAddr is -1:
return 1
# Allocate the exec_bo
execHandle = xclAllocBO(opt.handle, opt.DATA_SIZE, xclBOKind.XCL_BO_SHARED_VIRTUAL, (1 << 31))
execData = xclMapBO(opt.handle, execHandle, True) # returns mmap()
c_f = ffi.cast("FILE *", execData)
if execData is ffi.NULL:
print("execData is NULL")
print("Construct the exe buf cmd to configure FPGA")
ecmd = ert_configure_cmd()
ecmd.m_uert.m_cmd_struct.state = 1 # ERT_CMD_STATE_NEW
ecmd.m_uert.m_cmd_struct.opcode = 2 # ERT_CONFIGURE
ecmd.slot_size = opt.DATA_SIZE
ecmd.num_cus = 1
ecmd.cu_shift = 16
ecmd.cu_base_addr = opt.cu_base_addr
ecmd.m_features.ert = opt.ert
if opt.ert:
ecmd.m_features.cu_dma = 1
ecmd.m_features.cu_isr = 1
# CU -> base address mapping
ecmd.data[0] = opt.cu_base_addr
ecmd.m_uert.m_cmd_struct.count = 5 + ecmd.num_cus
sz = sizeof(ert_configure_cmd)
ffi.memmove(c_f, ecmd, sz)
print("Send the exec command and configure FPGA (ERT)")
# Send the command.
ret = xclExecBuf(opt.handle, execHandle)
if ret:
print("Unable to issue xclExecBuf")
return 1
print("Wait until the command finish")
while xclExecWait(opt.handle, 1000) != 0:
print(".")
print("Construct the exec command to run the kernel on FPGA")
# construct the exec buffer cmd to start the kernel
start_cmd = ert_start_kernel_cmd()
rsz = (XHELLO_HELLO_CONTROL_ADDR_ACCESS1_DATA / 4 + 1) + 1 # regmap array size
new_data = ((start_cmd.data._type_) * rsz)()
start_cmd.m_uert.m_start_cmd_struct.state = 1 # ERT_CMD_STATE_NEW
start_cmd.m_uert.m_start_cmd_struct.opcode = 0 # ERT_START_CU
start_cmd.m_uert.m_start_cmd_struct.count = 1 + rsz
start_cmd.cu_mask = 0x1
new_data[XHELLO_HELLO_CONTROL_ADDR_AP_CTRL] = 0x0
new_data[XHELLO_HELLO_CONTROL_ADDR_ACCESS1_DATA / 4] = bodevAddr
new_data[XHELLO_HELLO_CONTROL_ADDR_ACCESS1_DATA / 4 + 1] = (bodevAddr >> 32) & 0xFFFFFFFF
ffi.memmove(c_f, start_cmd, 2 * sizeof(c_uint32))
tmp_buf = ffi.buffer(c_f, 2 * sizeof(c_uint32) + (len(new_data) * sizeof(c_uint32)))
data_ptr = ffi.from_buffer(tmp_buf)
ffi.memmove(data_ptr + 2 * sizeof(c_uint32), new_data, len(new_data) * sizeof(c_uint32))
if xclExecBuf(opt.handle, execHandle):
print("Unable to issue xclExecBuf")
return 1
else:
print("Kernel start command issued through xclExecBuf : start_kernel")
print("Now wait until the kernel finish")
print("Wait until the command finish")
while xclExecWait(opt.handle, 1) != 0:
print(".")
# get the output xclSyncBO
print("Get the output data from the device")
if xclSyncBO(opt.handle, boHandle, xclBOSyncDirection.XCL_BO_SYNC_BO_FROM_DEVICE, opt.DATA_SIZE, 0):
return 1
rd_buf = ffi.buffer(read_fp, len("Hello World"))
print("RESULT: ")
print(rd_buf[:] + "\n")
return 0
elif os.path.exists("./libdistributions.so"):
lib = ffi.dlopen("./libdistributions.so")
else:
raise RuntimeError("Required DLL/so file was not found.")
ffi.cdef("""
double random_standard_normal(void *bitgen_state);
""")
x = PCG64()
xffi = x.cffi
bit_generator = xffi.bit_generator
random_standard_normal = lib.random_standard_normal
def normals(n, bit_generator):
out = np.empty(n)
for i in range(n):
out[i] = random_standard_normal(bit_generator)
return out
normalsj = nb.jit(normals, nopython=True)
# Numba requires a memory address for void *
# Can also get address from x.ctypes.bit_generator.value
bit_generator_address = int(ffi.cast("uintptr_t", bit_generator))
norm = normalsj(1000, bit_generator_address)
print(norm[:12])
{int64:s}
{int32:s}
""".format_map(signatures)
print(sig_text)
ffi.cdef(sig_text)
#
# open the library file
#
lib = ffi.dlopen(lib_so_file)
print('found these functions in module: ', dir(lib))
make_unique_double = lib.make_unique_double
type_dict = dict(double=np.float64, float=np.float32, int=np.int64)
in_vec = np.array([0, 5, 5, 1, 2, 2, 2, 3, 4, 4, 4, 4, 4, 5], dtype=np.float64)
invec_ptr = ffi.cast("double *", in_vec.ctypes.data)
invec_len = in_vec.size
type_dict = {
'double': (np.float64, lib.make_unique_double),
'float': (np.float32, lib.make_unique_float),
'int64_t': (np.int64, lib.make_unique_int64),
'int32_t': (np.int32, lib.make_unique_int32)
}
in_vec = np.array([0, 5, 5, 1, 2, 2, 2, 3, 4, 4, 4, 4, 4, 5])
print(f'original vector: {in_vec}')
for type_key, value in type_dict.items():
print(f'testing {type_key}')
np_type, the_fun = value
in_vec = np.array([0, 5, 5, 1, 2, 2, 2, 3, 4, 4, 4, 4, 4, 5],
class QuadraticMarchingCubes:
def __init__(self):
self.ffi = FFI()
with open("Src/exported_routines.h") as header:
header_str = header.read()
cstr = ""
ignore = False
for line in header_str.splitlines():
if (line.startswith("#if")):
ignore = True
if (ignore == False):
cstr += line
if (line.startswith("#end")):
ignore = False
self.ffi.cdef(cstr)
correctWorkingDirectory = os.getcwd()
libname_start = correctWorkingDirectory + "/build/libquadratic_iso"
if (platform.system() == "Darwin"):
if os.path.exists(libname_start + ".dylib"):
libname = libname_start + ".dylib"
else:
libname = libname_start + "d.dylib"
elif (platform.system() == "Windows"):
libname = correctWorkingDirectory + "/build/Debug/quadratic_iso.dll"
else:
if os.path.exists(libname_start + ".so"):
libname = libname_start + ".so"
else:
libname = libname_start + "d.so"
self.isosurf = self.ffi.dlopen(libname)
os.chdir(os.getcwd())
print(self.isosurf)
def run(self, isovalue, np_sdf_data, dim, path=None):
#Allocate the maximum possible amount of memory used for buffers
#passed into the C++ code.
np_tris = np.zeros((dim * dim * dim, 3), dtype=np.int32)
np_verts = np.zeros((dim * dim * dim, 3), dtype=np.float32)
ffi_vert_count = self.ffi.new("int*")
ffi_tri_count = self.ffi.new("int*")
#Run the C++ code.
self.isosurf.run_quadratic_mc(
self.ffi.cast("int", dim),
self.ffi.cast("float*", np_sdf_data.ctypes.data),
self.ffi.cast("float", isovalue),
self.ffi.cast("float*", np_verts.ctypes.data), ffi_vert_count,
self.ffi.cast("int*", np_tris.ctypes.data), ffi_tri_count)
#Trim off unused memory
np_verts = np_verts[:ffi_vert_count[0], :]
np_tris = np_tris[:ffi_tri_count[0], :]
if path is not None:
with open(path, "w") as f:
f.write("# OBJ file\n")
for i in range(ffi_vert_count[0]):
f.write(
f"v {np_verts[i,0]:3.4f} {np_verts[i,1]:3.4f} {np_verts[i,2]:3.4f}\n"
)
for i in range(ffi_tri_count[0]):
f.write(
f"f {np_tris[i,0]+1:d} {np_tris[i,1]+1:d} {np_tris[i,2]+1:d}\n"
)
return np_verts, np_tris
bin_params.globalBoundary = GLOBAL_BOUNDARY
bin_params.localBoundary = LOCAL_BOUNDARY
bin_params.fieldReach = FIELD_REACH
bin_params.fieldSize = FIELD_SIZE
preTime = time.time() - preStart
rustStart = time.time()
rust_rtn = C.segment_buffer(c_arr,arr.shape[1],arr.shape[0],bin_params)
rustTime = time.time() - rustStart
postStart = time.time()
pixels_arr = rust_rtn.pixels
print("SIZE:",pixels_arr.size)
pixels = ffi.cast("uint8_t*",pixels_arr.ptr)
debug_arr = np.array([pixels[i] for i in range(pixels_arr.size)]).astype("uint8")
# print("debug_arr.dtype:",debug_arr.dtype)
debug_img = np.reshape(debug_arr,arr.shape,'C')
# print(debug_img.shape)
img = Image.fromarray(debug_img, 'RGB')
img = Image.fromarray(debug_img, 'RGB')
postTime = time.time() - postStart
print("Time:",f"{preTime:.3f}","->",f"{rustTime:.3f}","->",f"{postTime:.3f}")
img.save("test_image.png")
img.show()
lines_arr = rust_rtn.symbols
def setup_function(function):
print("setting up %s" % function)
if __name__ == "__main__":
global shm
import sys
shmstep0()
print shm.fd, shm.size, ffi.string(shm.name)
c = sys.stdin.readline()
print "readlins is return", c
if c[:4] == "open":
print "into openex test process 2"
shmstep1x()
print "copy 7 bytes "
caddr = ffi.cast("void*", shm.addr)
C.memcpy(caddr, "welcome", 7)
sys.stdin.readline()
print "copy %d bytes" % (4096 * 2 + 1)
C.memcpy(caddr, "welcome" * 2000, 4096 * 2 + 1)
sys.stdin.readline()
shmstep2()
elif c[:5] == "close":
pass
#C.shm_unlink(shm.name)
else:
shmstep1()
print "into open test process 1"
sys.stdin.readline()
shmstep2()
def test_vi_C_interface():
try:
from cffi import FFI
cffi_is_present = True
except:
cffi_is_present = False
return
if cffi_is_present:
h = 1e-5
T = 1.0
t = 0.0
theta = 1.0
gamma = 1.0
g = 9.81
kappa = 0.4
xk = np.array((1., 10.))
ffi = FFI()
ffi.cdef('void set_cstruct(uintptr_t p_env, void* p_struct);')
ffi.cdef('''typedef struct
{
int id;
double* xk;
double h;
double theta;
double gamma;
double g;
double kappa;
} data;
''')
data_struct = ffi.new('data*')
data_struct.id = -1 # to avoid freeing the data in the destructor
data_struct.xk = ffi.cast('double *', xk.ctypes.data)
data_struct.h = h
data_struct.theta = theta
data_struct.gamma = gamma
data_struct.g = g
data_struct.kappa = kappa
vi = SN.VI(2)
import siconos
D = ffi.dlopen(siconos.__path__[0] + '/_numerics.so')
D.set_cstruct(vi.get_env_as_long(), ffi.cast('void*', data_struct))
vi.set_compute_F_and_nabla_F_as_C_functions('ZhuravlevIvanov.so', 'compute_F', 'compute_nabla_F')
lambda_ = np.zeros((2,))
xkp1 = np.zeros((2,))
SO = SN.SolverOptions(vi, SN.SICONOS_VI_BOX_QI)
lb = np.array((-1.0, -1.0))
ub = np.array((1.0, 1.0))
vi.set_box_constraints(lb, ub)
N = int(T/h + 10)
print(N)
SO.dparam[0] = 1e-24
SO.iparam[0] = 100
SO.iparam[2] = 1
SO.iparam[3] = 0
SO.iparam[4] = 5
signs = np.empty((N, 2))
sol = np.empty((N, 2))
sol[0, :] = xk
k = 0
#SN.setNumericsVerbose(3)
while t <= T:
k += 1
info = SN.variationalInequality_box_newton_QiLSA(vi, lambda_, xkp1, SO)
#print('iter {:} ; solver iter = {:} ; prec = {:}'.format(k, SO.iparam[1], SO.dparam[1]))
if info > 0:
print(lambda_)
# vi_function(2, signs[k-1, :], xkp1)
lambda_[0] = -np.sign(xkp1[0])
lambda_[1] = -np.sign(xkp1[1])
if np.abs(xk[0]) < 1e-10:
lambda_[0] = 0.01
if np.abs(xk[1]) < 1e-10:
lambda_[1] = 0.01
print('ok lambda')
print(lambda_)
info = SN.variationalInequality_box_newton_QiLSA(vi, lambda_, xkp1, SO)
print('iter {:} ; solver iter = {:} ; prec = {:}'.format(k, SO.iparam[1], SO.dparam[1]))
if info >0:
print('VI solver failed ! info = {:}'.format(info))
print(xk)
print(lambda_)
print(xkp1)
kaboom()
# else:
# print('iter {:} ; solver iter = {:} ; prec = {:}'.format(k, SO.iparam[1], SO.dparam[1]))
# vi_function(2, lambda_, xkp1)
sol[k, 0:2] = xkp1
np.copyto(xk, xkp1, casting='no')
signs[k, 0:2] = lambda_
t = k*h
from ctypes import c_double, c_int, pointer, POINTER
import numpy as np
ffi = FFI()
chi2 = ffi.dlopen('./_chi2.so')
ffi.cdef("""
int chi2(double m, double b, double *x, double *y, double *yerr, int N, double* result);
""")
if __name__ == '__main__':
mu_y = 0.
sig_y = 5.
mu_x = 0.21
sig_x = 3.
N = int(1000)
m = float(1.)
b = float(0.)
x = np.random.normal(mu_x, sig_x, N).astype('float64')
x_p = ffi.cast('double *', x.ctypes.data)
y = np.random.normal(mu_y, sig_y, N).astype('float64')
y_p = ffi.cast('double *', y.ctypes.data)
yerr = np.array([sig_y] * N, 'float64')
yerr_p = ffi.cast('double *', yerr.ctypes.data)
result = ffi.new('double *')
chi2.chi2(m, b, x_p, y_p, yerr_p, N, result)
print(result[0])
class Image(object):
def __init__(self,
input_image,
greyscale=False,
matrix=None,
depth=KM_UINT8):
"""
input_image: An image.
greyscale: A boolean indicating whether to read the image on a greyscale or not
matrix: An array representing the image.
depth: An integer indicating each channel depth
"""
np_type = {
KM_FLOAT32: np.float32,
KM_INT32: np.int32,
KM_UINT8: np.uint8
}.get(depth, None)
if type(matrix) is np.ndarray:
np_type == matrix.dtype
elif matrix == None:
pass
else:
raise ValueError('Only numpy arrays are allowed.')
if np_type not in [np.uint8, np.int32, np.float32]:
raise ValueError('Invalid type for numpy array.')
self.ffi = FFI()
with open('./src/includes/wrapper.h', 'r') as f:
self.ffi.cdef(f.read())
self.lib = self.ffi.dlopen('./wrapper.so')
self.input_image = input_image
self.greyscale = greyscale
self.np_type = np_type
self.depth = depth
self.matrix = matrix or self.read_image()
(self.height, self.width) = self.matrix.shape[:2]
def read_image(self):
"""
Read input_image using OpenCV (uses L*a*b* color space)
"""
image = cv2.imread(self.input_image, 0 if self.greyscale else 1)
if not self.greyscale:
image = cv2.cvtColor(image, cv2.COLOR_BGR2LAB)
if self.np_type == np.float32:
image = image.astype(self.np_type) / np.iinfo(image.dtype).max
return image
def posterize(self, levels):
"""
Posterize image using KMeans with levels number of clusters
"""
points = self.height * self.width
dim = 1 if self.greyscale else 3
image = self.matrix.flatten()
# Cast numpy array to void *
c_matrix = self.ffi.cast('void *', image.ctypes.data)
# Apply KMeans and get centroids
c_centroids = self.lib.kmeans(c_matrix, points, dim, levels,
self.depth)
# Paint posterized image
c_posterized = self.lib.assign_color(c_matrix, c_centroids, points,
dim, levels, self.depth)
# Cast posterized image to it original type
pointer_type = {
np.float32: 'float *',
np.int32: 'int32_t *',
np.uint8: 'uint8_t *'
}[self.np_type]
c_posterized_cast = self.ffi.cast(pointer_type, c_posterized)
# Initialize buffer from which we'll read the new numpy array
c_buffer = self.ffi.buffer(c_posterized_cast,
points * dim * self.matrix.dtype.itemsize)
posterized = np.frombuffer(c_buffer, dtype=self.np_type)
if self.np_type == np.float32:
posterized = (posterized * 255).astype(np.uint8)
# Reshape it to the original size
if self.greyscale:
posterized = np.reshape(posterized, (self.height, self.width))
else:
posterized = np.reshape(posterized, (self.height, self.width, 3))
posterized = cv2.cvtColor(posterized, cv2.COLOR_LAB2BGR)
output = re.sub('(?P<pre>.*)(?P<format>\..*?)',
'\g<pre>_output\g<format>', self.input_image)
cv2.imwrite(output, posterized)
