"""Global constants for the flux interface""" import cffi __CAST_ONLY_FFI = cffi.FFI() #constants from #define values FLUX_NODEID_ANY = int(__CAST_ONLY_FFI.cast('uint32_t', ~0)) FLUX_NODEID_UPSTREAM = int(__CAST_ONLY_FFI.cast('uint32_t', ~1)) FLUX_MSGTYPE_REQUEST = 0x01 FLUX_MSGTYPE_RESPONSE = 0x02 FLUX_MSGTYPE_EVENT = 0x04 FLUX_MSGTYPE_KEEPALIVE = 0x08 FLUX_MSGTYPE_ANY = 0x0f FLUX_MSGTYPE_MASK = 0x0f FLUX_MSGFLAG_TOPIC = 0x01 FLUX_MSGFLAG_PAYLOAD = 0x02 FLUX_MSGFLAG_JSON = 0x04 FLUX_MSGFLAG_ROUTE = 0x08 FLUX_MSGFLAG_UPSTREAM = 0x10 FLUX_O_TRACE = 1 FLUX_O_COPROC = 2 FLUX_O_NONBLOCK = 4 FLUX_SEC_TYPE_PLAIN = 1 FLUX_SEC_TYPE_CURVE = 2 FLUX_SEC_TYPE_MUNGE = 4 FLUX_SEC_TYPE_ALL = 7
import os if os.path.exists('VkInline/cffi.py'): os.remove('VkInline/cffi.py') import cffi ffibuilder = cffi.FFI() ffibuilder.set_source("VkInline.cffi", None) ffibuilder.cdef(""" // utils void* n_string_array_create(unsigned long long size, const char* const* strs); unsigned long long n_string_array_size(void* ptr_arr); void n_string_array_destroy(void* ptr_arr); void* n_pointer_array_create(unsigned long long size, const void* const* ptrs); unsigned long long n_pointer_array_size(void* ptr_arr); void n_pointer_array_destroy(void* ptr_arr); void* n_dim3_create(unsigned x, unsigned y, unsigned z); void n_dim3_destroy(void* cptr); // Context int n_vkinline_try_init(); void n_set_verbose(unsigned verbose); unsigned long long n_size_of(const char* cls); void n_add_built_in_header(const char* filename, const char* filecontent); void n_add_inlcude_filename(const char* fn); void n_add_code_block(const char* line); void n_wait(); void* n_computer_create(void* ptr_param_list, const char* body); void n_computer_destroy(void* cptr); int n_computer_num_params(void* cptr);
def main(): ffibuilder = cffi.FFI() ffibuilder.embedding_api(""" void tenkei_library_language( uint8_t *input, size_t input_len, uint8_t **output, size_t *output_len ); void tenkei_binary_or( uint8_t *input, size_t input_len, uint8_t **output, size_t *output_len ); void tenkei_modify_array( uint8_t *input, size_t input_len, uint8_t **output, size_t *output_len ); void tenkei_exponentiate( uint8_t *input, size_t input_len, uint8_t **output, size_t *output_len ); void tenkei_identity( uint8_t *input, size_t input_len, uint8_t **output, size_t *output_len ); void tenkei_choose_left( uint8_t *input, size_t input_len, uint8_t **output, size_t *output_len ); void tenkei_reverse_list( uint8_t *input, size_t input_len, uint8_t **output, size_t *output_len ); void tenkei_apply_function( uint8_t *input, size_t input_len, uint8_t **output, size_t *output_len ); void tenkei_free( uint8_t *buffer, size_t buffer_len ); """) ffibuilder.set_source("my_plugin", "") ffibuilder.embedding_init_code(read_file("../libtenkei-python/ffi_wrappers.py") + "\n\n" + """ @ffi.def_extern() def tenkei_free(buffer, buffer_len): DONT_FORGET[buffer] -= 1 if DONT_FORGET[buffer] == 0: del DONT_FORGET[buffer] @ffi.def_extern() def tenkei_library_language(*args): return offer_with_conversion(library_language, [])(*args) @ffi.def_extern() def tenkei_binary_or(*args): return offer_with_conversion(binary_or, [])(*args) @ffi.def_extern() def tenkei_modify_array(*args): return offer_with_conversion(modify_array, [])(*args) @ffi.def_extern() def tenkei_exponentiate(*args): return offer_with_conversion(exponentiate, [])(*args) @ffi.def_extern() def tenkei_identity(*args): return offer_with_conversion(identity, [])(*args) @ffi.def_extern() def tenkei_choose_left(*args): return offer_with_conversion(choose_left, [])(*args) @ffi.def_extern() def tenkei_reverse_list(*args): return offer_with_conversion(reverse_list, [])(*args) @ffi.def_extern() def tenkei_apply_function(*args): return offer_with_conversion(apply_function, [0])(*args) """ + read_file("test-library.py")) ffibuilder.compile(target="libtest-library.*")
def setUp(self): self.ffi = cffi.FFI() with open("output/s2hell.h.preprocessed") as f: self.ffi.cdef(f.read()) self.lib = self.ffi.dlopen("output/x86/s2hell.so")
def _initialize(backends): global lib if lib is not None: return # C function definitions # size_t instead of void* is used # for convinience with python calls and numpy arrays, # cffi automatically calls int() on objects also. src = """ typedef int cudnnStatus_t; typedef size_t cudnnHandle_t; typedef size_t cudnnTensorDescriptor_t; typedef size_t cudnnConvolutionDescriptor_t; typedef size_t cudnnFilterDescriptor_t; typedef size_t cudnnPoolingDescriptor_t; typedef int cudnnTensorFormat_t; typedef int cudnnDataType_t; typedef int cudnnConvolutionMode_t; typedef int cudnnConvolutionFwdPreference_t; typedef int cudnnConvolutionFwdAlgo_t; typedef int cudnnPoolingMode_t; typedef int cudnnSoftmaxAlgorithm_t; typedef int cudnnSoftmaxMode_t; size_t cudnnGetVersion(); cudnnStatus_t cudnnCreate(cudnnHandle_t *handle); cudnnStatus_t cudnnDestroy(cudnnHandle_t handle); cudnnStatus_t cudnnCreateTensorDescriptor( cudnnTensorDescriptor_t *tensorDesc); cudnnStatus_t cudnnDestroyTensorDescriptor( cudnnTensorDescriptor_t tensorDesc); cudnnStatus_t cudnnSetTensor4dDescriptor( cudnnTensorDescriptor_t tensorDesc, cudnnTensorFormat_t format, cudnnDataType_t dataType, int n, int c, int h, int w); cudnnStatus_t cudnnGetTensor4dDescriptor( const cudnnTensorDescriptor_t tensorDesc, cudnnDataType_t *dataType, int *n, int *c, int *h, int *w, int *nStride, int *cStride, int *hStride, int *wStride); cudnnStatus_t cudnnSetTensorNdDescriptor( cudnnTensorDescriptor_t tensorDesc, cudnnDataType_t dataType, int nbDims, const int *dimA, const int *strideA); cudnnStatus_t cudnnGetTensorNdDescriptor( const cudnnTensorDescriptor_t tensorDesc, int nbDimsRequested, cudnnDataType_t *dataType, int *nbDims, int *dimA, int *strideA); cudnnStatus_t cudnnCreateFilterDescriptor( cudnnFilterDescriptor_t *filterDesc); cudnnStatus_t cudnnDestroyFilterDescriptor( cudnnFilterDescriptor_t filterDesc); cudnnStatus_t cudnnCreateConvolutionDescriptor( cudnnConvolutionDescriptor_t *convDesc); cudnnStatus_t cudnnDestroyConvolutionDescriptor( cudnnConvolutionDescriptor_t convDesc); cudnnStatus_t cudnnSetConvolution2dDescriptor( cudnnConvolutionDescriptor_t convDesc, int pad_h, int pad_w, int u, int v, int upscalex, int upscaley, cudnnConvolutionMode_t mode); cudnnStatus_t cudnnGetConvolution2dForwardOutputDim( const cudnnConvolutionDescriptor_t convDesc, const cudnnTensorDescriptor_t inputTensorDesc, const cudnnFilterDescriptor_t filterDesc, int *n, int *c, int *h, int *w); cudnnStatus_t cudnnGetConvolutionForwardAlgorithm( cudnnHandle_t handle, const cudnnTensorDescriptor_t srcDesc, const cudnnFilterDescriptor_t filterDesc, const cudnnConvolutionDescriptor_t convDesc, const cudnnTensorDescriptor_t destDesc, cudnnConvolutionFwdPreference_t preference, size_t memoryLimitInbytes, cudnnConvolutionFwdAlgo_t *algo); cudnnStatus_t cudnnGetConvolutionForwardWorkspaceSize( cudnnHandle_t handle, const cudnnTensorDescriptor_t srcDesc, const cudnnFilterDescriptor_t filterDesc, const cudnnConvolutionDescriptor_t convDesc, const cudnnTensorDescriptor_t destDesc, cudnnConvolutionFwdAlgo_t algo, size_t *sizeInBytes); cudnnStatus_t cudnnConvolutionForward( cudnnHandle_t handle, const intptr_t alpha, const cudnnTensorDescriptor_t srcDesc, const intptr_t srcData, const cudnnFilterDescriptor_t filterDesc, const intptr_t filterData, const cudnnConvolutionDescriptor_t convDesc, cudnnConvolutionFwdAlgo_t algo, intptr_t workSpace, size_t workSpaceSizeInBytes, const intptr_t beta, const cudnnTensorDescriptor_t destDesc, intptr_t destData); cudnnStatus_t cudnnConvolutionBackwardBias( cudnnHandle_t handle, const intptr_t alpha, const cudnnTensorDescriptor_t srcDesc, const intptr_t srcData, const intptr_t beta, const cudnnTensorDescriptor_t destDesc, intptr_t destData); cudnnStatus_t cudnnTransformTensor( cudnnHandle_t handle, const intptr_t alpha, const cudnnTensorDescriptor_t srcDesc, const intptr_t srcData, const intptr_t beta, const cudnnTensorDescriptor_t destDesc, intptr_t destData); cudnnStatus_t cudnnCreatePoolingDescriptor( cudnnPoolingDescriptor_t *poolingDesc); cudnnStatus_t cudnnDestroyPoolingDescriptor( cudnnPoolingDescriptor_t poolingDesc); cudnnStatus_t cudnnGetPooling2dForwardOutputDim( const cudnnPoolingDescriptor_t poolingDesc, const cudnnTensorDescriptor_t inputTensorDesc, int *n, int *c, int *h, int *w); cudnnStatus_t cudnnPoolingForward( cudnnHandle_t handle, const cudnnPoolingDescriptor_t poolingDesc, const intptr_t alpha, const cudnnTensorDescriptor_t xDesc, const intptr_t x, const intptr_t beta, const cudnnTensorDescriptor_t yDesc, intptr_t y); cudnnStatus_t cudnnPoolingBackward( cudnnHandle_t handle, const cudnnPoolingDescriptor_t poolingDesc, const intptr_t alpha, const cudnnTensorDescriptor_t yDesc, const intptr_t y, const cudnnTensorDescriptor_t dyDesc, const intptr_t dy, const cudnnTensorDescriptor_t xDesc, const intptr_t x, const intptr_t beta, const cudnnTensorDescriptor_t dxDesc, intptr_t dx); cudnnStatus_t cudnnSoftmaxForward( cudnnHandle_t handle, cudnnSoftmaxAlgorithm_t algo, cudnnSoftmaxMode_t mode, const intptr_t alpha, const cudnnTensorDescriptor_t xDesc, const intptr_t x, const intptr_t beta, const cudnnTensorDescriptor_t yDesc, intptr_t y); cudnnStatus_t cudnnSoftmaxBackward( cudnnHandle_t handle, cudnnSoftmaxAlgorithm_t algo, cudnnSoftmaxMode_t mode, const intptr_t alpha, const cudnnTensorDescriptor_t yDesc, const intptr_t y, const cudnnTensorDescriptor_t dyDesc, const intptr_t dy, const intptr_t beta, const cudnnTensorDescriptor_t dxDesc, intptr_t dx); """ src2 = """ cudnnStatus_t cudnnConvolutionBackwardFilter( cudnnHandle_t handle, const intptr_t alpha, const cudnnTensorDescriptor_t srcDesc, const intptr_t srcData, const cudnnTensorDescriptor_t diffDesc, const intptr_t diffData, const cudnnConvolutionDescriptor_t convDesc, const intptr_t beta, const cudnnFilterDescriptor_t gradDesc, intptr_t gradData); cudnnStatus_t cudnnConvolutionBackwardData( cudnnHandle_t handle, const intptr_t alpha, const cudnnFilterDescriptor_t filterDesc, const intptr_t filterData, const cudnnTensorDescriptor_t diffDesc, const intptr_t diffData, const cudnnConvolutionDescriptor_t convDesc, const intptr_t beta, const cudnnTensorDescriptor_t gradDesc, intptr_t gradData); """ src4p = """ typedef int cudnnConvolutionBwdFilterAlgo_t; typedef int cudnnConvolutionBwdDataAlgo_t; typedef int cudnnConvolutionBwdFilterPreference_t; typedef int cudnnConvolutionBwdDataPreference_t; cudnnStatus_t cudnnGetConvolutionBackwardFilterAlgorithm( cudnnHandle_t handle, const cudnnTensorDescriptor_t xDesc, const cudnnTensorDescriptor_t dyDesc, const cudnnConvolutionDescriptor_t convDesc, const cudnnFilterDescriptor_t dwDesc, cudnnConvolutionBwdFilterPreference_t preference, size_t memoryLimitInBytes, cudnnConvolutionBwdFilterAlgo_t *algo); cudnnStatus_t cudnnGetConvolutionBackwardFilterWorkspaceSize( cudnnHandle_t handle, const cudnnTensorDescriptor_t xDesc, const cudnnTensorDescriptor_t dyDesc, const cudnnConvolutionDescriptor_t convDesc, const cudnnFilterDescriptor_t gradDesc, cudnnConvolutionBwdFilterAlgo_t algo, size_t *sizeInBytes); cudnnStatus_t cudnnConvolutionBackwardFilter( cudnnHandle_t handle, const intptr_t alpha, const cudnnTensorDescriptor_t srcDesc, const intptr_t srcData, const cudnnTensorDescriptor_t diffDesc, const intptr_t diffData, const cudnnConvolutionDescriptor_t convDesc, const cudnnConvolutionBwdFilterAlgo_t algo, intptr_t workSpace, size_t workSpaceSizeInBytes, const intptr_t beta, const cudnnFilterDescriptor_t gradDesc, intptr_t gradData); cudnnStatus_t cudnnGetConvolutionBackwardDataAlgorithm( cudnnHandle_t handle, const cudnnFilterDescriptor_t wDesc, const cudnnTensorDescriptor_t dyDesc, const cudnnConvolutionDescriptor_t convDesc, const cudnnTensorDescriptor_t dxDesc, cudnnConvolutionBwdDataPreference_t preference, size_t memoryLimitInBytes, cudnnConvolutionBwdDataAlgo_t *algo); cudnnStatus_t cudnnGetConvolutionBackwardDataWorkspaceSize( cudnnHandle_t handle, const cudnnFilterDescriptor_t wDesc, const cudnnTensorDescriptor_t dyDesc, const cudnnConvolutionDescriptor_t convDesc, const cudnnTensorDescriptor_t dxDesc, cudnnConvolutionBwdDataAlgo_t algo, size_t *sizeInBytes); cudnnStatus_t cudnnConvolutionBackwardData( cudnnHandle_t handle, const intptr_t alpha, const cudnnFilterDescriptor_t filterDesc, const intptr_t filterData, const cudnnTensorDescriptor_t diffDesc, const intptr_t diffData, const cudnnConvolutionDescriptor_t convDesc, const cudnnConvolutionBwdDataAlgo_t algo, intptr_t workSpace, size_t workSpaceSizeInBytes, const intptr_t beta, const cudnnTensorDescriptor_t gradDesc, intptr_t gradData); """ src24 = """ cudnnStatus_t cudnnSetFilter4dDescriptor( cudnnFilterDescriptor_t filterDesc, cudnnDataType_t dataType, int k, int c, int h, int w); cudnnStatus_t cudnnSetPooling2dDescriptor( cudnnPoolingDescriptor_t poolingDesc, cudnnPoolingMode_t mode, int windowHeight, int windowWidth, int verticalPadding, int horizontalPadding, int verticalStride, int horizontalStride); """ src5 = """ typedef size_t cudnnRNNDescriptor_t; typedef size_t cudnnDropoutDescriptor_t; typedef int cudnnNanPropagation_t; typedef int cudnnRNNInputMode_t; typedef int cudnnDirectionMode_t; typedef int cudnnRNNMode_t; cudnnStatus_t cudnnSetFilter4dDescriptor( cudnnFilterDescriptor_t filterDesc, cudnnDataType_t dataType, cudnnTensorFormat_t format, int k, int c, int h, int w); cudnnStatus_t cudnnGetFilter4dDescriptor( const cudnnFilterDescriptor_t filterDesc, cudnnDataType_t *dataType, cudnnTensorFormat_t *format, int *k, int *c, int *h, int *w); cudnnStatus_t cudnnSetFilterNdDescriptor( cudnnFilterDescriptor_t filterDesc, cudnnDataType_t dataType, cudnnTensorFormat_t format, int nbDims, const int *filterDimA); cudnnStatus_t cudnnGetFilterNdDescriptor( const cudnnFilterDescriptor_t filterDesc, int nbDimsRequested, cudnnDataType_t *dataType, cudnnTensorFormat_t *format, int *nbDims, int *filterDimA); cudnnStatus_t cudnnSetPooling2dDescriptor( cudnnPoolingDescriptor_t poolingDesc, cudnnPoolingMode_t mode, cudnnNanPropagation_t maxpoolingNanOpt, int windowHeight, int windowWidth, int verticalPadding, int horizontalPadding, int verticalStride, int horizontalStride); cudnnStatus_t cudnnCreateDropoutDescriptor( cudnnDropoutDescriptor_t *dropoutDesc); cudnnStatus_t cudnnDestroyDropoutDescriptor( cudnnDropoutDescriptor_t dropoutDesc); cudnnStatus_t cudnnDropoutGetStatesSize( cudnnHandle_t handle, size_t *sizeInBytes); cudnnStatus_t cudnnDropoutGetReserveSpaceSize( cudnnTensorDescriptor_t xdesc, size_t *sizeInBytes); cudnnStatus_t cudnnSetDropoutDescriptor( cudnnDropoutDescriptor_t dropoutDesc, cudnnHandle_t handle, float dropout, intptr_t states, size_t stateSizeInBytes, unsigned long long seed); cudnnStatus_t cudnnDropoutForward( cudnnHandle_t handle, const cudnnDropoutDescriptor_t dropoutDesc, const cudnnTensorDescriptor_t xdesc, const intptr_t x, const cudnnTensorDescriptor_t ydesc, intptr_t y, intptr_t reserveSpace, size_t reserveSpaceSizeInBytes); cudnnStatus_t cudnnDropoutBackward( cudnnHandle_t handle, const cudnnDropoutDescriptor_t dropoutDesc, const cudnnTensorDescriptor_t dydesc, const intptr_t dy, const cudnnTensorDescriptor_t dxdesc, intptr_t dx, intptr_t reserveSpace, size_t reserveSpaceSizeInBytes); cudnnStatus_t cudnnCreateRNNDescriptor(cudnnRNNDescriptor_t *rnnDesc); cudnnStatus_t cudnnDestroyRNNDescriptor(cudnnRNNDescriptor_t rnnDesc); cudnnStatus_t cudnnSetRNNDescriptor( cudnnRNNDescriptor_t rnnDesc, int hiddenSize, int numLayers, cudnnDropoutDescriptor_t dropoutDesc, cudnnRNNInputMode_t inputMode, cudnnDirectionMode_t direction, cudnnRNNMode_t mode, cudnnDataType_t dataType); cudnnStatus_t cudnnGetRNNWorkspaceSize( cudnnHandle_t handle, const cudnnRNNDescriptor_t rnnDesc, const int seqLength, const cudnnTensorDescriptor_t *xDesc, size_t *sizeInBytes); cudnnStatus_t cudnnGetRNNTrainingReserveSize( cudnnHandle_t handle, const cudnnRNNDescriptor_t rnnDesc, const int seqLength, const cudnnTensorDescriptor_t *xDesc, size_t *sizeInBytes); cudnnStatus_t cudnnGetRNNParamsSize( cudnnHandle_t handle, const cudnnRNNDescriptor_t rnnDesc, const cudnnTensorDescriptor_t xDesc, size_t *sizeInBytes, cudnnDataType_t dataType); cudnnStatus_t cudnnGetRNNLinLayerMatrixParams( cudnnHandle_t handle, const cudnnRNNDescriptor_t rnnDesc, const int layer, const cudnnTensorDescriptor_t xDesc, const cudnnFilterDescriptor_t wDesc, const intptr_t w, const int linLayerID, cudnnFilterDescriptor_t linLayerMatDesc, intptr_t *linLayerMat); cudnnStatus_t cudnnGetRNNLinLayerBiasParams( cudnnHandle_t handle, const cudnnRNNDescriptor_t rnnDesc, const int layer, const cudnnTensorDescriptor_t xDesc, const cudnnFilterDescriptor_t wDesc, const intptr_t w, const int linLayerID, cudnnFilterDescriptor_t linLayerBiasDesc, intptr_t *linLayerBias); cudnnStatus_t cudnnRNNForwardInference( cudnnHandle_t handle, const cudnnRNNDescriptor_t rnnDesc, const int seqLength, const cudnnTensorDescriptor_t *xDesc, const intptr_t x, const cudnnTensorDescriptor_t hxDesc, const intptr_t hx, const cudnnTensorDescriptor_t cxDesc, const intptr_t cx, const cudnnFilterDescriptor_t wDesc, const intptr_t w, const cudnnTensorDescriptor_t *yDesc, intptr_t y, const cudnnTensorDescriptor_t hyDesc, intptr_t hy, const cudnnTensorDescriptor_t cyDesc, intptr_t cy, intptr_t workspace, size_t workSpaceSizeInBytes); cudnnStatus_t cudnnRNNForwardTraining( cudnnHandle_t handle, const cudnnRNNDescriptor_t rnnDesc, const int seqLength, const cudnnTensorDescriptor_t *xDesc, const intptr_t x, const cudnnTensorDescriptor_t hxDesc, const intptr_t hx, const cudnnTensorDescriptor_t cxDesc, const intptr_t cx, const cudnnFilterDescriptor_t wDesc, const intptr_t w, const cudnnTensorDescriptor_t *yDesc, intptr_t y, const cudnnTensorDescriptor_t hyDesc, intptr_t hy, const cudnnTensorDescriptor_t cyDesc, intptr_t cy, intptr_t workspace, size_t workSpaceSizeInBytes, intptr_t reserveSpace, size_t reserveSpaceSizeInBytes); cudnnStatus_t cudnnRNNBackwardData( cudnnHandle_t handle, const cudnnRNNDescriptor_t rnnDesc, const int seqLength, const cudnnTensorDescriptor_t *yDesc, const intptr_t y, const cudnnTensorDescriptor_t *dyDesc, const intptr_t dy, const cudnnTensorDescriptor_t dhyDesc, const intptr_t dhy, const cudnnTensorDescriptor_t dcyDesc, const intptr_t dcy, const cudnnFilterDescriptor_t wDesc, const intptr_t w, const cudnnTensorDescriptor_t hxDesc, const intptr_t hx, const cudnnTensorDescriptor_t cxDesc, const intptr_t cx, const cudnnTensorDescriptor_t *dxDesc, intptr_t dx, const cudnnTensorDescriptor_t dhxDesc, intptr_t dhx, const cudnnTensorDescriptor_t dcxDesc, intptr_t dcx, intptr_t workspace, size_t workSpaceSizeInBytes, const intptr_t reserveSpace, size_t reserveSpaceSizeInBytes); cudnnStatus_t cudnnRNNBackwardWeights( cudnnHandle_t handle, const cudnnRNNDescriptor_t rnnDesc, const int seqLength, const cudnnTensorDescriptor_t *xDesc, const intptr_t x, const cudnnTensorDescriptor_t hxDesc, const intptr_t hx, const cudnnTensorDescriptor_t *yDesc, const intptr_t y, const intptr_t workspace, size_t workSpaceSizeInBytes, const cudnnFilterDescriptor_t dwDesc, intptr_t dw, const intptr_t reserveSpace, size_t reserveSpaceSizeInBytes); """ # Parse global ffi ffi = cffi.FFI() ffi.cdef(src) # Load library for libnme in backends: try: lib = ffi.dlopen(libnme) break except OSError: pass else: ffi = None raise OSError("Could not load cudnn library") global cudnn_version cudnn_version = lib.cudnnGetVersion() if cudnn_version < 4000: ffi.cdef(src2) # specific functions for V2 ffi.cdef(src24) # specific functions for V2 and V4 else: ffi.cdef(src4p) # specific functions for V4+ if cudnn_version < 5000: ffi.cdef(src24) # specific functions for V2 and V4 else: ffi.cdef(src5) # specific functions for V5 global ERRORS for code, msg in ERRORS.items(): if code in CU.ERRORS: s = " | " + msg if s not in CU.ERRORS[code]: CU.ERRORS[code] += s else: CU.ERRORS[code] = msg
def create_extension(name, headers, sources, verbose=True, with_cuda=False, package=False, relative_to='.', **kwargs): """Creates and configures a cffi.FFI object, that builds PyTorch extension. Arguments: name (str): package name. Can be a nested module e.g. ``.ext.my_lib``. headers (str or List[str]): list of headers, that contain only exported functions sources (List[str]): list of sources to compile. verbose (bool, optional): if set to ``False``, no output will be printed (default: True). with_cuda (bool, optional): set to ``True`` to compile with CUDA headers (default: False) package (bool, optional): set to ``True`` to build in package mode (for modules meant to be installed as pip packages) (default: False). relative_to (str, optional): path of the build file. Required when ``package is True``. It's best to use ``__file__`` for this argument. kwargs: additional arguments that are passed to ffi to declare the extension. See `Extension API reference`_ for details. .. _`Extension API reference`: https://docs.python.org/3/distutils/apiref.html#distutils.core.Extension """ base_path = os.path.abspath(os.path.dirname(relative_to)) name_suffix, target_dir = _create_module_dir(base_path, name) if not package: cffi_wrapper_name = '_' + name_suffix else: cffi_wrapper_name = (name.rpartition('.')[0] + '.{0}._{0}'.format(name_suffix)) wrapper_source, include_dirs = _setup_wrapper(with_cuda) include_dirs.extend(kwargs.pop('include_dirs', [])) if os.sys.platform == 'win32': library_dirs = glob.glob(os.getenv('CUDA_PATH', '') + '/lib/x64') library_dirs += glob.glob(os.getenv('NVTOOLSEXT_PATH', '') + '/lib/x64') here = os.path.abspath(os.path.dirname(__file__)) lib_dir = os.path.join(here, '..', '..', 'lib') library_dirs.append(os.path.join(lib_dir)) else: library_dirs = [] library_dirs.extend(kwargs.pop('library_dirs', [])) if isinstance(headers, str): headers = [headers] all_headers_source = '' for header in headers: with open(os.path.join(base_path, header), 'r') as f: all_headers_source += f.read() + '\n\n' ffi = cffi.FFI() sources = [os.path.join(base_path, src) for src in sources] ffi.set_source(cffi_wrapper_name, wrapper_source + all_headers_source, sources=sources, include_dirs=include_dirs, library_dirs=library_dirs, **kwargs) ffi.cdef(_typedefs + all_headers_source) _make_python_wrapper(name_suffix, '_' + name_suffix, target_dir) def build(): _build_extension(ffi, cffi_wrapper_name, target_dir, verbose) ffi.build = build return ffi
def __init__(self, form: ufl.Form, form_compiler_parameters: dict = None): """Create dolfinx Form Parameters ---------- form Pure UFL form form_compiler_parameters Parameters used in JIT FFCX compilation of this form Note ---- This wrapper for UFL form is responsible for the actual FFCX compilation and attaching coefficients and domains specific data to the underlying C++ Form. """ self.form_compiler_parameters = form_compiler_parameters # Extract subdomain data from UFL form sd = form.subdomain_data() self._subdomains, = list(sd.values()) # Assuming single domain domain, = list(sd.keys()) # Assuming single domain mesh = domain.ufl_cargo() # Compile UFL form with JIT ufc_form = jit.ffcx_jit( form, form_compiler_parameters=self.form_compiler_parameters, mpi_comm=mesh.mpi_comm()) # For every argument in form extract its function space function_spaces = [ func.ufl_function_space()._cpp_object for func in form.arguments() ] # Prepare dolfinx.Form and hold it as a member ffi = cffi.FFI() self._cpp_object = cpp.fem.create_form(ffi.cast("uintptr_t", ufc_form), function_spaces) # Need to fill the form with coefficients data # For every coefficient in form take its C++ object original_coefficients = form.coefficients() for i in range(self._cpp_object.num_coefficients()): j = self._cpp_object.original_coefficient_position(i) self._cpp_object.set_coefficient( i, original_coefficients[j]._cpp_object) # Constants are set based on their position in original form original_constants = [c._cpp_object for c in form.constants()] self._cpp_object.set_constants(original_constants) if mesh is None: raise RuntimeError("Expecting to find a Mesh in the form.") # Attach mesh (because function spaces and coefficients may be # empty lists) if not function_spaces: self._cpp_object.set_mesh(mesh) # Attach subdomains to C++ Form if we have them subdomains = self._subdomains.get("cell") if subdomains: self._cpp_object.set_cell_domains(subdomains) subdomains = self._subdomains.get("exterior_facet") if subdomains: self._cpp_object.set_exterior_facet_domains(subdomains) subdomains = self._subdomains.get("interior_facet") if subdomains: self._cpp_object.set_interior_facet_domains(subdomains) subdomains = self._subdomains.get("vertex") if subdomains: self._cpp_object.set_vertex_domains(subdomains)
def _initialize(backends): global lib if lib is not None: return # C function definitions src = """ typedef int32_t cl_int; typedef uint32_t cl_uint; typedef uint64_t cl_ulong; typedef uint64_t cl_device_type; typedef uint32_t cl_platform_info; typedef uint32_t cl_device_info; typedef uint32_t cl_program_build_info; typedef cl_uint cl_program_info; typedef cl_uint cl_kernel_info; typedef uint32_t cl_kernel_work_group_info; typedef uint64_t cl_command_queue_properties; typedef uint64_t cl_queue_properties; typedef uint64_t cl_mem_flags; typedef uint32_t cl_bool; typedef uint64_t cl_map_flags; typedef uint32_t cl_profiling_info; typedef uint32_t cl_buffer_create_type; typedef uint64_t cl_svm_mem_flags; typedef void* cl_platform_id; typedef void* cl_device_id; typedef void* cl_context; typedef void* cl_program; typedef void* cl_kernel; typedef void* cl_command_queue; typedef void* cl_mem; typedef void* cl_event; typedef intptr_t cl_context_properties; typedef intptr_t cl_pipe_properties; cl_int clGetPlatformIDs(cl_uint num_entries, cl_platform_id *platforms, cl_uint *num_platforms); cl_int clGetDeviceIDs(cl_platform_id platform, cl_device_type device_type, cl_uint num_entries, cl_device_id *devices, cl_uint *num_devices); cl_int clGetPlatformInfo(cl_platform_id platform, cl_platform_info param_name, size_t param_value_size, void *param_value, size_t *param_value_size_ret); cl_int clGetDeviceInfo(cl_device_id device, cl_device_info param_name, size_t param_value_size, void *param_value, size_t *param_value_size_ret); cl_context clCreateContext(const cl_context_properties *properties, cl_uint num_devices, const cl_device_id *devices, void *pfn_notify, void *user_data, cl_int *errcode_ret); cl_int clReleaseContext(cl_context context); cl_program clCreateProgramWithSource(cl_context context, cl_uint count, const char **strings, const size_t *lengths, cl_int *errcode_ret); cl_program clCreateProgramWithBinary(cl_context context, cl_uint num_devices, const cl_device_id *device_list, const size_t *lengths, const unsigned char **binaries, cl_int *binary_status, cl_int *errcode_ret); cl_int clReleaseProgram(cl_program program); cl_int clBuildProgram(cl_program program, cl_uint num_devices, const cl_device_id *device_list, const char *options, void *pfn_notify, void *user_data); cl_int clGetProgramBuildInfo(cl_program program, cl_device_id device, cl_program_build_info param_name, size_t param_value_size, void *param_value, size_t *param_value_size_ret); cl_int clGetProgramInfo(cl_program program, cl_program_info param_name, size_t param_value_size, void *param_value, size_t *param_value_size_ret); cl_kernel clCreateKernel(cl_program program, const char *kernel_name, cl_int *errcode_ret); cl_int clReleaseKernel(cl_kernel kernel); cl_int clGetKernelInfo(cl_kernel kernel, cl_kernel_info param_name, size_t param_value_size, void *param_value, size_t *param_value_size_ret); cl_int clGetKernelWorkGroupInfo(cl_kernel kernel, cl_device_id device, cl_kernel_work_group_info param_name, size_t param_value_size, void *param_value, size_t *param_value_size_ret); cl_int clSetKernelArg(cl_kernel kernel, cl_uint arg_index, size_t arg_size, const void *arg_value); cl_command_queue clCreateCommandQueue( cl_context context, cl_device_id device, cl_command_queue_properties properties, cl_int *errcode_ret); cl_command_queue clCreateCommandQueueWithProperties( cl_context context, cl_device_id device, const cl_queue_properties *properties, cl_int *errcode_ret); cl_int clReleaseCommandQueue(cl_command_queue command_queue); cl_mem clCreateBuffer(cl_context context, cl_mem_flags flags, size_t size, void *host_ptr, cl_int *errcode_ret); cl_mem clCreateSubBuffer(cl_mem buffer, cl_mem_flags flags, cl_buffer_create_type buffer_create_type, const void *buffer_create_info, cl_int *errcode_ret); cl_int clReleaseMemObject(cl_mem memobj); void* clEnqueueMapBuffer(cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_map, cl_map_flags map_flags, size_t offset, size_t size, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event, cl_int *errcode_ret); cl_int clEnqueueUnmapMemObject(cl_command_queue command_queue, cl_mem memobj, void *mapped_ptr, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event); cl_int clEnqueueReadBuffer(cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_read, size_t offset, size_t size, void *ptr, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event); cl_int clEnqueueWriteBuffer(cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_write, size_t offset, size_t size, const void *ptr, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event); cl_int clEnqueueCopyBuffer(cl_command_queue command_queue, cl_mem src_buffer, cl_mem dst_buffer, size_t src_offset, size_t dst_offset, size_t size, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event); cl_int clEnqueueCopyBufferRect(cl_command_queue command_queue, cl_mem src_buffer, cl_mem dst_buffer, const size_t *src_origin, const size_t *dst_origin, const size_t *region, size_t src_row_pitch, size_t src_slice_pitch, size_t dst_row_pitch, size_t dst_slice_pitch, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event); cl_int clEnqueueFillBuffer(cl_command_queue command_queue, cl_mem buffer, const void *pattern, size_t pattern_size, size_t offset, size_t size, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event); cl_int clWaitForEvents(cl_uint num_events, const cl_event *event_list); cl_int clReleaseEvent(cl_event event); cl_int clFlush(cl_command_queue command_queue); cl_int clFinish(cl_command_queue command_queue); cl_int clEnqueueNDRangeKernel(cl_command_queue command_queue, cl_kernel kernel, cl_uint work_dim, const size_t *global_work_offset, const size_t *global_work_size, const size_t *local_work_size, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event); cl_int clGetEventProfilingInfo(cl_event event, cl_profiling_info param_name, size_t param_value_size, void *param_value, size_t *param_value_size_ret); cl_mem clCreatePipe(cl_context context, cl_mem_flags flags, cl_uint pipe_packet_size, cl_uint pipe_max_packets, const cl_pipe_properties *properties, cl_int *errcode_ret); void *clSVMAlloc(cl_context context, cl_svm_mem_flags flags, size_t size, unsigned int alignment); void clSVMFree(cl_context context, void *svm_pointer); cl_int clEnqueueSVMMap(cl_command_queue command_queue, cl_bool blocking_map, cl_map_flags map_flags, void *svm_ptr, size_t size, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event); cl_int clEnqueueSVMUnmap(cl_command_queue command_queue, void *svm_ptr, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event); cl_int clSetKernelArgSVMPointer(cl_kernel kernel, cl_uint arg_index, const void *arg_value); cl_int clEnqueueSVMMemcpy(cl_command_queue command_queue, cl_bool blocking_copy, void *dst_ptr, const void *src_ptr, size_t size, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event); cl_int clEnqueueSVMMemFill(cl_command_queue command_queue, void *svm_ptr, const void *pattern, size_t pattern_size, size_t size, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event); """ # Parse global ffi ffi = cffi.FFI() ffi.cdef(src) # Load library for libnme in backends: try: lib = ffi.dlopen(libnme) break except OSError: pass else: ffi = None raise OSError("Could not load OpenCL library")
def ffi_init(lib_name, cffi_cdef, headers=[], libraries=[], include_dirs=[cuda_include_path], library_dirs=[cuda_lib_path], **kwargs): """ initialize FFI and FFILibrary objects using cffi Parameters ---------- lib_name : str name of the cffi Out-of-Line module to be passed to ffi.set_source() cffi_cdef : str cffi cdef string headers : list of str, optional list of additional headers to include in the call to ffi.verify libraries : list of str, optional list of library names to pass to ffi.verify include_dirs : list of str, optional list of include paths for ffi.verify library_dirs : list of str, optional list of library paths for ffi.verify Returns ------- ffi : cffi.api.FFI object cffi FFI object returned by ffi.verify ffi_lib : cffi.vengine_cpy.FFILibrary cffi FFILibrary object returned by ffi.verify """ ffi = cffi.FFI() ffi.cdef(cffi_cdef) c_header_source = '' for hdr in headers: c_header_source += "#include <{}>\n".format(hdr) if cffi.__version__ > '1': use_verify = False # now depricated, so no longer use it else: use_verify = True # now depricated, so no longer use it if use_verify: ffi_lib = ffi.verify(c_header_source, libraries=libraries, include_dirs=include_dirs, library_dirs=library_dirs, **kwargs) return ffi, ffi_lib else: previous_dir = os.getcwd() os.chdir(os.path.dirname(__file__)) ffi.set_source(lib_name, c_header_source, libraries=libraries, include_dirs=include_dirs, library_dirs=library_dirs, **kwargs) ffi.compile() os.chdir(previous_dir) mod = importlib.import_module('.' + lib_name, package='cuda_cffi') return mod.ffi, mod.lib
def build_ffi(): ffi = cffi.FFI() ffi.cdef( textwrap.dedent(""" typedef unsigned char CK_BYTE; typedef unsigned long CK_ULONG; typedef unsigned long CK_RV; typedef unsigned long CK_SESSION_HANDLE; typedef unsigned long CK_OBJECT_HANDLE; typedef unsigned long CK_SLOT_ID; typedef unsigned long CK_FLAGS; typedef unsigned long CK_STATE; typedef unsigned long CK_USER_TYPE; typedef unsigned char * CK_UTF8CHAR_PTR; typedef ... *CK_NOTIFY; typedef unsigned long ck_attribute_type_t; struct ck_attribute { ck_attribute_type_t type; void *value; unsigned long value_len; }; typedef struct ck_attribute CK_ATTRIBUTE; typedef CK_ATTRIBUTE *CK_ATTRIBUTE_PTR; typedef unsigned long ck_mechanism_type_t; struct ck_mechanism { ck_mechanism_type_t mechanism; void *parameter; unsigned long parameter_len; }; typedef struct ck_mechanism CK_MECHANISM; typedef CK_MECHANISM *CK_MECHANISM_PTR; typedef CK_BYTE *CK_BYTE_PTR; typedef CK_ULONG *CK_ULONG_PTR; typedef struct ck_session_info { CK_SLOT_ID slot_id; CK_STATE state; CK_FLAGS flags; unsigned long device_error; } CK_SESSION_INFO; typedef CK_SESSION_INFO *CK_SESSION_INFO_PTR; typedef struct CK_AES_GCM_PARAMS { char * pIv; unsigned long ulIvLen; unsigned long ulIvBits; char * pAAD; unsigned long ulAADLen; unsigned long ulTagBits; } CK_AES_GCM_PARAMS; """)) # FUNCTIONS ffi.cdef( textwrap.dedent(""" CK_RV C_Initialize(void *); CK_RV C_Finalize(void *); CK_RV C_OpenSession(CK_SLOT_ID, CK_FLAGS, void *, CK_NOTIFY, CK_SESSION_HANDLE *); CK_RV C_CloseSession(CK_SESSION_HANDLE); CK_RV C_GetSessionInfo(CK_SESSION_HANDLE, CK_SESSION_INFO_PTR); CK_RV C_Login(CK_SESSION_HANDLE, CK_USER_TYPE, CK_UTF8CHAR_PTR, CK_ULONG); CK_RV C_SetAttributeValue(CK_SESSION_HANDLE, CK_OBJECT_HANDLE, CK_ATTRIBUTE *, CK_ULONG); CK_RV C_DestroyObject(CK_SESSION_HANDLE, CK_OBJECT_HANDLE); CK_RV C_FindObjectsInit(CK_SESSION_HANDLE, CK_ATTRIBUTE *, CK_ULONG); CK_RV C_FindObjects(CK_SESSION_HANDLE, CK_OBJECT_HANDLE *, CK_ULONG, CK_ULONG *); CK_RV C_FindObjectsFinal(CK_SESSION_HANDLE); CK_RV C_GenerateKey(CK_SESSION_HANDLE, CK_MECHANISM *, CK_ATTRIBUTE *, CK_ULONG, CK_OBJECT_HANDLE *); CK_RV C_UnwrapKey(CK_SESSION_HANDLE, CK_MECHANISM *, CK_OBJECT_HANDLE, CK_BYTE *, CK_ULONG, CK_ATTRIBUTE *, CK_ULONG, CK_OBJECT_HANDLE *); CK_RV C_WrapKey(CK_SESSION_HANDLE, CK_MECHANISM_PTR, CK_OBJECT_HANDLE, CK_OBJECT_HANDLE, CK_BYTE_PTR, CK_ULONG_PTR); CK_RV C_EncryptInit(CK_SESSION_HANDLE, CK_MECHANISM_PTR, CK_OBJECT_HANDLE); CK_RV C_Encrypt(CK_SESSION_HANDLE, CK_BYTE_PTR, CK_ULONG, CK_BYTE_PTR, CK_ULONG_PTR); CK_RV C_DecryptInit(CK_SESSION_HANDLE, CK_MECHANISM_PTR, CK_OBJECT_HANDLE); CK_RV C_Decrypt(CK_SESSION_HANDLE, CK_BYTE_PTR, CK_ULONG, CK_BYTE_PTR, CK_ULONG_PTR); CK_RV C_SignInit(CK_SESSION_HANDLE, CK_MECHANISM_PTR, CK_OBJECT_HANDLE); CK_RV C_Sign(CK_SESSION_HANDLE, CK_BYTE_PTR, CK_ULONG, CK_BYTE_PTR, CK_ULONG_PTR); CK_RV C_VerifyInit(CK_SESSION_HANDLE, CK_MECHANISM_PTR, CK_OBJECT_HANDLE); CK_RV C_Verify(CK_SESSION_HANDLE, CK_BYTE_PTR, CK_ULONG, CK_BYTE_PTR, CK_ULONG); CK_RV C_GenerateRandom(CK_SESSION_HANDLE, CK_BYTE_PTR, CK_ULONG); CK_RV C_SeedRandom(CK_SESSION_HANDLE, CK_BYTE_PTR, CK_ULONG); """)) return ffi
def __init__(self, source=None, snaplen=65535, promisc=True, to_ms=1, bpf_filter=None, nroots=1, account_ip_padding_size=False, decode_tunnels=False): errors = validate_parameters(source, promisc, snaplen, bpf_filter, account_ip_padding_size, decode_tunnels) if errors != '': raise OSError(errors) self._ffi = cffi.FFI() self._lib = self._ffi.dlopen( dirname(abspath(__file__)) + '/observer_cc.so') self._ffi.cdef(cc_observer_headers) self._ffi.cdef(cc_observer_apis, override=True) if source in net_if_addrs().keys(): self.mode = 1 # we found source in device interfaces and set mode to live. elif ".pcap" in source[-5:] and isfile(source): self.mode = 0 # .pcap extension and file exists, we set mode to offline else: raise OSError( "Undefined source: {}. " "Please specify a pcap file path or a valid network interface name." .format(source)) if self.mode in [0, 1]: error_buffer = self._ffi.new("char []", 128) error_buffer_setter = self._ffi.new("char []", 128) handler = self._lib.observer_open(bytes(source, 'utf-8'), snaplen, int(promisc), to_ms, error_buffer, error_buffer_setter, self.mode) if handler == self._ffi.NULL: raise OSError( self._ffi.string(error_buffer).decode('ascii', 'ignore') + "\n" + self._ffi.string(error_buffer_setter).decode( 'ascii', 'ignore')) else: # Once we have a valid handler, we move to BPF filtering configuration. if isinstance(bpf_filter, str): rs = self._lib.observer_configure( handler, bytes(bpf_filter, 'utf-8')) if rs > 0: raise OSError( "Failed to setup BPF filter: {}. Please use a valid one." .format(bpf_filter)) elif bpf_filter is None: pass else: raise OSError( "Please specify a pcap file path or a valid network interface name as source." ) self.cap = handler self.nroots = nroots self.safety_time = 0 self.account_ip_padding_size = account_ip_padding_size self.decode_tunnels = int(decode_tunnels)
def _get_cffi_data(): if not HAVE_CFFI: raise nose.SkipTest("cffi not installed") ffi = cffi.FFI() return ffi.new('double *', 2.0)
def __init__(self, form: ufl.Form, form_compiler_parameters: dict = {}, jit_parameters: dict = {}): """Create dolfinx Form Parameters ---------- form Pure UFL form form_compiler_parameters See :py:func:`ffcx_jit <dolfinx.jit.ffcx_jit>` jit_parameters See :py:func:`ffcx_jit <dolfinx.jit.ffcx_jit>` Note ---- This wrapper for UFL form is responsible for the actual FFCX compilation and attaching coefficients and domains specific data to the underlying C++ Form. """ # Extract subdomain data from UFL form sd = form.subdomain_data() self._subdomains, = list(sd.values()) # Assuming single domain domain, = list(sd.keys()) # Assuming single domain mesh = domain.ufl_cargo() if mesh is None: raise RuntimeError("Expecting to find a Mesh in the form.") # Compile UFL form with JIT ufc_form = jit.ffcx_jit( mesh.mpi_comm(), form, form_compiler_parameters=form_compiler_parameters, jit_parameters=jit_parameters) # For every argument in form extract its function space function_spaces = [ func.ufl_function_space()._cpp_object for func in form.arguments() ] # Prepare coefficients data. For every coefficient in form take # its C++ object. original_coefficients = form.coefficients() coeffs = [ original_coefficients[ ufc_form.original_coefficient_position[i]]._cpp_object for i in range(ufc_form.num_coefficients) ] # Create dictionary of of subdomain markers (possible None for # some dimensions subdomains = { cpp.fem.IntegralType.cell: self._subdomains.get("cell"), cpp.fem.IntegralType.exterior_facet: self._subdomains.get("exterior_facet"), cpp.fem.IntegralType.interior_facet: self._subdomains.get("interior_facet"), cpp.fem.IntegralType.vertex: self._subdomains.get("vertex") } # Prepare dolfinx.cpp.fem.Form and hold it as a member ffi = cffi.FFI() self._cpp_object = cpp.fem.create_form( ffi.cast("uintptr_t", ffi.addressof(ufc_form)), function_spaces, coeffs, [c._cpp_object for c in form.constants()], subdomains, mesh)
def __init__(self, n_components=2, perplexity=30.0, early_exaggeration=12, learning_rate=200, n_iter=1000, n_iter_early_exag=250, n_iter_without_progress=30, min_grad_norm=1e-07, metric='euclidean', init='random', verbose=0, random_state=None, method='barnes_hut', angle=0.5, n_jobs=1, cheat_metric=True, auto_iter=False, auto_iter_end=5000): self.n_components = n_components self.angle = angle self.perplexity = perplexity self.early_exaggeration = early_exaggeration self.learning_rate = learning_rate self.n_iter = n_iter self.n_iter_early_exag = n_iter_early_exag self.n_jobs = n_jobs self.random_state = -1 if random_state is None else random_state self.init = init self.embedding_ = None self.n_iter_ = None self.kl_divergence_ = None self.verbose = int(verbose) self.cheat_metric = cheat_metric self.auto_iter = auto_iter self.auto_iter_end = auto_iter_end assert isinstance( init, np.ndarray) or init == 'random', "init must be 'random' or array" if isinstance(init, np.ndarray): assert init.ndim == 2, "init array must be 2D" assert init.shape[ 1] == n_components, "init array must be of shape (n_instances, n_components)" self.init = np.ascontiguousarray(init, float) self.ffi = cffi.FFI() self.ffi.cdef( """void tsne_run_double(double* X, int N, int D, double* Y, int no_dims, double perplexity, double theta, int num_threads, int max_iter, int n_iter_early_exag, int random_state, bool init_from_Y, int verbose, double early_exaggeration, double learning_rate, double *final_error, int distance, bool auto_iter, double auto_iter_end);""" ) path = os.path.dirname(os.path.realpath(__file__)) try: sofile = (glob(os.path.join(path, 'libtsne*.so')) + glob(os.path.join(path, '*tsne*.dll')))[0] self.C = self.ffi.dlopen(os.path.join(path, sofile)) except (IndexError, OSError): raise RuntimeError( 'Cannot find/open tsne_multicore shared library')
def _initialize(backends): global lib if lib is not None: return # C function definitions src = """ typedef int32_t cl_int; typedef uint32_t cl_uint; typedef float cl_float; typedef double cl_double; typedef void* cl_mem; typedef void* cl_command_queue; typedef void* cl_event; typedef int clblasStatus; typedef int clblasOrder; typedef int clblasTranspose; clblasStatus clblasSetup(); void clblasTeardown(); clblasStatus clblasSgemm(clblasOrder order, clblasTranspose transA, clblasTranspose transB, size_t M, size_t N, size_t K, cl_float alpha, const cl_mem A, size_t offA, size_t lda, const cl_mem B, size_t offB, size_t ldb, cl_float beta, cl_mem C, size_t offC, size_t ldc, cl_uint numCommandQueues, cl_command_queue *commandQueues, cl_uint numEventsInWaitList, const cl_event *eventWaitList, cl_event *events); clblasStatus clblasDgemm(clblasOrder order, clblasTranspose transA, clblasTranspose transB, size_t M, size_t N, size_t K, cl_double alpha, const cl_mem A, size_t offA, size_t lda, const cl_mem B, size_t offB, size_t ldb, cl_double beta, cl_mem C, size_t offC, size_t ldc, cl_uint numCommandQueues, cl_command_queue *commandQueues, cl_uint numEventsInWaitList, const cl_event *eventWaitList, cl_event *events); """ # Parse global ffi ffi = cffi.FFI() ffi.cdef(src) # Load library for libnme in backends: try: lib = ffi.dlopen(libnme) break except OSError: pass else: ffi = None raise OSError("Could not load clBlas library") global ERRORS CL.ERRORS.update(ERRORS)
def test_abi_emit_python_code_2(self): ffi = cffi.FFI() ffi.set_source("package_name_1.mymod", None) py.test.raises(IOError, ffi.emit_python_code, 'unexisting/xyz.py')
kwargs = pkgconfig_kwargs ([ "libzmq", "libczmq", "libmlm" ]) import cffi # can't import does not work, read and exec manually with open (os.path.join ( os.path.dirname (__file__), "cdefs.py"), 'r') as fp: cdefs_py = fp.read() gl = {} exec (cdefs_py, gl) malamute_cdefs = gl ["malamute_cdefs"] ffibuilder = cffi.FFI () ffibuilder.set_source ("malamute_cffi.native", "#include <malamute.h>", **kwargs) # Custom setup for malamute for item in malamute_cdefs: ffibuilder.cdef(item) ffidestructorbuilder = cffi.FFI () ffidestructorbuilder.cdef(''' void mlm_proto_destroy_py (void *self); void mlm_client_destroy_py (void *self); ''')
def test_abi_emit_python_code_3(self): ffi = cffi.FFI() ffi.set_source("package_name_1.mymod", None) ffi.emit_python_code(str(self.udir.join('xyt.py'))) self.check_produced_files({'xyt.py': None})
def _make_ffi_from_dialect(dialect): ffi = cffi.FFI() ffi.cdef(""" long parse_line(char *rawline, long inputlength); """) d = { 'quotechar': ord(dialect.quotechar), 'quoting': int(dialect.quoting), 'skipinitialspace': int(dialect.skipinitialspace), 'delimiter': ord(dialect.delimiter), 'doublequote': int(dialect.doublequote), 'strict': int(dialect.strict), } if dialect.escapechar is not None: d['is_escape_char'] = '== %d' % ord(dialect.escapechar) else: d['is_escape_char'] = '&& 0' lib = ffi.verify(r''' typedef enum { START_RECORD, START_FIELD, ESCAPED_CHAR, IN_FIELD, IN_QUOTED_FIELD, ESCAPE_IN_QUOTED_FIELD, QUOTE_IN_QUOTED_FIELD, EAT_CRNL } ParserState; typedef enum { QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE } QuoteStyle; typedef struct { ParserState state; /* current CSV parse state */ char *field; /* build current field in here */ int field_size; /* size of allocated buffer */ int field_len; /* length of current field */ int numeric_field; /* treat field as numeric */ } ReaderObj; static void parse_add_char(ReaderObj *self, char c) { *self->field++ = c; } static void parse_save_field(ReaderObj *self) { *self->field++ = 0; } static int parse_process_char(ReaderObj *self, char c) { switch (self->state) { case START_RECORD: /* start of record */ if (c == '\0') /* empty line - return [] */ break; else if (c == '\n' || c == '\r') { self->state = EAT_CRNL; break; } /* normal character - handle as START_FIELD */ self->state = START_FIELD; /* fallthru */ case START_FIELD: /* expecting field */ if (c == '\n' || c == '\r' || c == '\0') { /* save empty field - return [fields] */ parse_save_field(self); self->state = (c == '\0' ? START_RECORD : EAT_CRNL); } else if (c == %(quotechar)d && %(quoting)d != QUOTE_NONE) { /* start quoted field */ self->state = IN_QUOTED_FIELD; } else if (c %(is_escape_char)s) { /* possible escaped character */ self->state = ESCAPED_CHAR; } else if (c == ' ' && %(skipinitialspace)d) /* ignore space at start of field */ ; else if (c == %(delimiter)d) { /* save empty field */ parse_save_field(self); } else { /* begin new unquoted field */ if (%(quoting)d == QUOTE_NONNUMERIC) self->numeric_field = 1; parse_add_char(self, c); self->state = IN_FIELD; } break; case ESCAPED_CHAR: if (c == '\0') c = '\n'; parse_add_char(self, c); self->state = IN_FIELD; break; case IN_FIELD: /* in unquoted field */ if (c == '\n' || c == '\r' || c == '\0') { /* end of line - return [fields] */ parse_save_field(self); self->state = (c == '\0' ? START_RECORD : EAT_CRNL); } else if (c %(is_escape_char)s) { /* possible escaped character */ self->state = ESCAPED_CHAR; } else if (c == %(delimiter)d) { /* save field - wait for new field */ parse_save_field(self); self->state = START_FIELD; } else { /* normal character - save in field */ parse_add_char(self, c); } break; case IN_QUOTED_FIELD: /* in quoted field */ if (c == '\0') ; else if (c %(is_escape_char)s) { /* Possible escape character */ self->state = ESCAPE_IN_QUOTED_FIELD; } else if (c == %(quotechar)d && %(quoting)d != QUOTE_NONE) { if (%(doublequote)d) { /* doublequote; " represented by "" */ self->state = QUOTE_IN_QUOTED_FIELD; } else { /* end of quote part of field */ self->state = IN_FIELD; } } else { /* normal character - save in field */ parse_add_char(self, c); } break; case ESCAPE_IN_QUOTED_FIELD: if (c == '\0') c = '\n'; parse_add_char(self, c); self->state = IN_QUOTED_FIELD; break; case QUOTE_IN_QUOTED_FIELD: /* doublequote - seen a quote in an quoted field */ if (%(quoting)d != QUOTE_NONE && c == %(quotechar)d) { /* save "" as " */ parse_add_char(self, c); self->state = IN_QUOTED_FIELD; } else if (c == %(delimiter)d) { /* save field - wait for new field */ parse_save_field(self); self->state = START_FIELD; } else if (c == '\n' || c == '\r' || c == '\0') { /* end of line - return [fields] */ parse_save_field(self); self->state = (c == '\0' ? START_RECORD : EAT_CRNL); } else if (!%(strict)d) { parse_add_char(self, c); self->state = IN_FIELD; } else { /* illegal */ /*PyErr_Format(error_obj, "'%%c' expected after '%%c'", dialect->delimiter, dialect->quotechar);*/ return -1; } break; case EAT_CRNL: if (c == '\n' || c == '\r') ; else if (c == '\0') self->state = START_RECORD; else { /*PyErr_Format(error_obj, "new-line character seen in unquoted field - do you need to open the file in universal-newline mode?");*/ return -1; } break; } return 0; } static void parse_reset(ReaderObj *self, char *rawline) { self->field = rawline; self->state = START_RECORD; self->numeric_field = 0; } long parse_line(char *rawline, long inputlength) { char *p; ReaderObj reader; parse_reset(&reader, rawline); for (p=rawline; inputlength > 0; inputlength--, p++) { if (parse_process_char(&reader, *p) < 0) return -1; } if (parse_process_char(&reader, 0) < 0) return -1; return reader.field - rawline - 1; } ''' % d) return ffi, lib
def test_abi_compile_1(self): ffi = cffi.FFI() ffi.set_source("mod_name_in_package.mymod", None) x = ffi.compile() self.check_produced_files({'mod_name_in_package': {'mymod.py': None}}) assert x == os.path.join('.', 'mod_name_in_package', 'mymod.py')
#LIDT declare LIDT_ROOT = os.getenv('LIDT_ROOT') LIDT_LIBS = os.path.sep.join((LIDT_ROOT, 'obj', 'libs')) JSON_BUILDER_FOR_C_INCLUDES = os.path.sep.join( (LIDT_ROOT, 'src', 'app', 'utils', 'jsonHelper', 'cWrapper', 'jsonBuilderWrapper')) JSON_BUILDER_FOR_C_LIBS = ["jsonBuilderForC"] #define common libraries BOOST_ROOT = os.getenv('BOOST_ROOT') BOOST_INCLUDES = os.path.sep.join((BOOST_ROOT, 'include')) BOOST_LIB_DIR = os.path.sep.join((BOOST_ROOT, 'lib')) BOOST_LIBS = ["boost_serialization", 'boost_thread', 'stdc++'] jsonBuilder_cffi = cffi.FFI() jsonBuilder_cffi.cdef(""" bool buildJsonForC(const char* message, char** jsonString); """) jsonBuilder_c = jsonBuilder_cffi.verify( """ #include "JsonBuilderForC.h" """, include_dirs=[JSON_BUILDER_FOR_C_INCLUDES, BOOST_INCLUDES], libraries=JSON_BUILDER_FOR_C_LIBS + BOOST_LIBS, library_dirs=[LIDT_LIBS, BOOST_LIB_DIR], modulename="__cffi_jsonBuilder")
def test_api_emit_c_code_2(self): ffi = cffi.FFI() ffi.set_source("package_name_1.mymod", "/*code would be here*/") py.test.raises(IOError, ffi.emit_c_code, 'unexisting/xyz.c')
def generate_pythonscriptcffi(output, cdef_path, dev_dyn): ffibuilder = cffi.FFI() # Divide between functions declarations and struct definitions with open('%s/api.h' % BASEDIR, 'r') as fd: api_src = fd.read() with open('%s/api_struct.h' % BASEDIR, 'r') as fd: api_struct_src = fd.read() # Def needed to compile output .c file ffibuilder.set_source( "pythonscriptcffi", """ #include <gdnative_api_struct.gen.h> // TODO: MethodFlags not in ldscript headers enum MethodFlags { METHOD_FLAG_NORMAL=1, METHOD_FLAG_EDITOR=2, METHOD_FLAG_NOSCRIPT=4, METHOD_FLAG_CONST=8, METHOD_FLAG_REVERSE=16, // used for events METHOD_FLAG_VIRTUAL=32, METHOD_FLAG_FROM_SCRIPT=64, METHOD_FLAG_VARARG=128, METHOD_FLAGS_DEFAULT=METHOD_FLAG_NORMAL, }; """ + api_src #) + """ static godot_real (*ptrfunc_godot_vector2_distance_to)(const godot_vector2 *p_self, const godot_vector2 *p_to) = godot_vector2_distance_to; typedef struct { godot_real (*_godot_vector2_distance_to)(const godot_vector2 *p_self, const godot_vector2 *p_to); } structfunc_t; static structfunc_t structfunc = { ._godot_vector2_distance_to=godot_vector2_distance_to }; godot_real structfunc_godot_vector2_distance_to(const godot_vector2 *p_self, const godot_vector2 *p_to) { return structfunc._godot_vector2_distance_to(p_self, p_to); } godot_real staticfunc_godot_vector2_distance_to(const godot_vector2 *p_self, const godot_vector2 *p_to) { return ptrfunc_godot_vector2_distance_to(p_self, p_to); } """) # Python source code embedded and run at init time # (including python functions exposed to C through `@ffi.def_extern()`) EMBEDDING_INC_SOURCES = ( 'embedding_init_code.inc.py', 'profiler.inc.py', 'allocator.inc.py', 'mod_godot.inc.py', 'builtin_rid.inc.py', 'builtin_color.inc.py', 'builtin_node_path.inc.py', 'builtin_vector2.inc.py', 'builtin_vector3.inc.py', 'builtin_plane.inc.py', 'builtin_rect2.inc.py', 'builtin_aabb.inc.py', 'builtin_basis.inc.py', 'builtin_quat.inc.py', 'builtin_transform2d.inc.py', 'builtin_transform.inc.py', 'builtin_pool_arrays.inc.py', 'builtin_array.inc.py', 'builtin_dictionary.inc.py', 'tools.inc.py', 'mod_godot_bindings.inc.py') # Hack not to have to compile everytime we modify an `*.inc.py` file if dev_dyn: embedding_init_code = """ code = [] for to_include in {sources!r}: with open('%s/%s' % ({basedir!r}, to_include), 'r') as fd: code.append(fd.read()) exec('\\n'.join(code)) """.format(sources=EMBEDDING_INC_SOURCES, basedir=BASEDIR) else: embedding_init_code = [] for to_include in EMBEDDING_INC_SOURCES: with open('%s/%s' % (BASEDIR, to_include), 'r') as fd: embedding_init_code.append(fd.read()) embedding_init_code = '\n'.join(embedding_init_code) ffibuilder.embedding_init_code(embedding_init_code) # C API exposed to Python with open(cdef_path) as fd: cdef = fd.read() ffibuilder.cdef(""" // TODO: not in ldscript headers ? enum MethodFlags { METHOD_FLAG_NORMAL=1, METHOD_FLAG_EDITOR=2, METHOD_FLAG_NOSCRIPT=4, METHOD_FLAG_CONST=8, METHOD_FLAG_REVERSE=16, // used for events METHOD_FLAG_VIRTUAL=32, METHOD_FLAG_FROM_SCRIPT=64, METHOD_FLAG_VARARG=128, METHOD_FLAGS_DEFAULT=METHOD_FLAG_NORMAL, }; // We use malloc to bypass Python garbage collector for Godot Object void *malloc(size_t size); void free(void *ptr); """ + cdef + strip_hashed_src(api_struct_src) + """ extern godot_real (*ptrfunc_godot_vector2_distance_to)(const godot_vector2 *p_self, const godot_vector2 *p_to); extern godot_real structfunc_godot_vector2_distance_to(const godot_vector2 *p_self, const godot_vector2 *p_to); extern godot_real staticfunc_godot_vector2_distance_to(const godot_vector2 *p_self, const godot_vector2 *p_to); typedef struct { godot_real (*_godot_vector2_distance_to)(const godot_vector2 *p_self, const godot_vector2 *p_to); } structfunc_t; extern structfunc_t structfunc; """) # Python `@ffi.def_extern()` API exposed to C ffibuilder.embedding_api(strip_hashed_src(api_src)) # Output .c code ready to be compiled ;-) ffibuilder.emit_c_code(output)
def test_api_emit_c_code_3(self): ffi = cffi.FFI() ffi.set_source("package_name_1.mymod", "/*code would be here*/") ffi.emit_c_code(str(self.udir.join('xyu.c'))) self.check_produced_files({'xyu.c': None})
def test_rank0(): """Test evaluation of UFL expression. This test evaluates gradient of P2 function at vertices of reference triangle. Because these points coincide with positions of point evaluation degrees-of-freedom of vector P1 space, values could be used to interpolate the expression into this space. This test also shows simple Numba assembler which accepts the donor P2 function ``f`` as a coefficient and tabulates vector P1 function into tensor ``b``. For a donor function f(x, y) = x^2 + 2*y^2 result is compared with the exact gradient grad f(x, y) = [2*x, 4*y]. """ mesh = dolfinx.generation.UnitSquareMesh(MPI.COMM_WORLD, 5, 5) P2 = dolfinx.FunctionSpace(mesh, ("P", 2)) vP1 = dolfinx.VectorFunctionSpace(mesh, ("P", 1)) f = dolfinx.Function(P2) def expr1(x): return x[0]**2 + 2.0 * x[1]**2 f.interpolate(expr1) ufl_expr = ufl.grad(f) points = np.array([[0.0, 0.0], [1.0, 0.0], [0.0, 1.0]]) compiled_expr = dolfinx.jit.ffcx_jit(mesh.mpi_comm(), (ufl_expr, points)) ffi = cffi.FFI() @numba.njit def assemble_expression(b, kernel, mesh, dofmap, coeff, coeff_dofmap): pos, x_dofmap, x = mesh geometry = np.zeros((3, 2)) w = np.zeros(6, dtype=PETSc.ScalarType) constants = np.zeros(1, dtype=PETSc.ScalarType) b_local = np.zeros(6, dtype=PETSc.ScalarType) for i, cell in enumerate(pos[:-1]): num_vertices = pos[i + 1] - pos[i] c = x_dofmap[cell:cell + num_vertices] for j in range(3): for k in range(2): geometry[j, k] = x[c[j], k] for j in range(6): w[j] = coeff[coeff_dofmap[i * 6 + j]] b_local.fill(0.0) kernel(ffi.from_buffer(b_local), ffi.from_buffer(w), ffi.from_buffer(constants), ffi.from_buffer(geometry)) for j in range(3): for k in range(2): b[2 * dofmap[i * 3 + j] + k] = b_local[2 * j + k] # Prepare mesh and dofmap data pos = mesh.geometry.dofmap.offsets x_dofs = mesh.geometry.dofmap.array x = mesh.geometry.x coeff_dofmap = P2.dofmap.list.array dofmap = vP1.dofmap.list.array # Data structure for the result b = dolfinx.Function(vP1) assemble_expression(b.vector.array, compiled_expr.tabulate_expression, (pos, x_dofs, x), dofmap, f.vector.array, coeff_dofmap) def grad_expr1(x): values = np.empty((2, x.shape[1])) values[0] = 2.0 * x[0] values[1] = 4.0 * x[1] return values b2 = dolfinx.Function(vP1) b2.interpolate(grad_expr1) assert np.isclose((b2.vector - b.vector).norm(), 0.0)
def test_abi_emit_python_code_1(self): ffi = cffi.FFI() ffi.set_source("package_name_1.mymod", None) ffi.emit_python_code('xyz.py') self.check_produced_files({'xyz.py': None})
import subprocess import sys import cffi if __name__ == "__main__": path_include_dir = sys.argv[1] path_library_dir = sys.argv[2] lib_cffi = cffi.FFI() preprocess = subprocess.run( "gcc -E -P %s/{{cookiecutter.lib_slug}}/*.h" % path_include_dir, check=True, shell=True, stdout=subprocess.PIPE, universal_newlines=True, ) csource = preprocess.stdout csource = csource[csource.find( "typedef void {{cookiecutter.lib_slug.upper()}}_UNUSED;"):] csource = "\n".join([csource, """ """]) lib_cffi.cdef(csource) lib_cffi.set_source( "_{{cookiecutter.lib_slug}}_cffi", """ #include "{{cookiecutter.lib_slug}}/{{cookiecutter.lib_slug}}.h" """, libraries=["{{cookiecutter.lib_slug}}"],
""" Preparation script for cffi module wrapping libsodium. """ import sys import os import os.path import cffi # Build the FFI instance. sodium_ffi = cffi.FFI() with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sodium_ffi.h'), 'r', encoding='utf-8') as sodium_ffi_h: sodium_ffi.cdef(sodium_ffi_h.read()) sodium_ffi.set_source( '_sodium', ( ('#define SODIUM_STATIC\n' if sys.platform == 'win32' else '') + \ '#include <sodium.h>' ), libraries=['libsodium' if sys.platform == 'win32' else 'sodium'] )
from math import sin,cos,radians Sin=lambda s:sin(radians(s)) Cos=lambda s:cos(radians(s)) import cffi,subprocess ffi=cffi.FFI() t=subprocess.Popen(['gcc','-E',"/tmp/matrix.h"],stdout=subprocess.PIPE, stderr=subprocess.PIPE) t.wait() dta=t.stdout.read().replace('\r\n','\n') ffi.cdef(dta) matrix=ffi.dlopen(__file__.rsplit('/',1)[0]+'/_matrix.so') class Matrix: def __init__(self, floats=None): if floats is None: floats = [1.000000,0.000000,0.000000,0.000000,0.000000,1.000000,0.000000,0.000000,0.000000,0.000000,1.000000,0.000000,0.000000,0.000000,0.000000,1.000000] self._matrix=matrix.matrixFromFloats(floats) def update(self, floats): matrix.updateMatrixFromFloats(self._matrix, floats) def __iter__(self): f=matrix.floatsFromMatrix(self._matrix) return iter((f+i)[0] for i in range(16)) def scale(self, x,y,z): matrix.scaleMatrix(self._matrix,x,y,z) def rotate(self, x,y,z,aboutCenter=True): if aboutCenter: center=self.getTranslation() scale=self.getScale() self.translate(-center[0],-center[1],-center[2]) self.scale(1/scale[0],1/scale[1],1/scale[2])
def _initialize(backends): global lib if lib is not None: return # C function definitions # size_t instead of void* is used # for convinience with python calls and numpy arrays. src = """ typedef int cublasStatus_t; typedef void *cublasHandle_t; typedef int cublasOperation_t; typedef int cublasPointerMode_t; typedef int cublasDataType_t; cublasStatus_t cublasCreate_v2(cublasHandle_t *handle); cublasStatus_t cublasDestroy_v2(cublasHandle_t handle); cublasStatus_t cublasSgemm_v2( cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, int m, int n, int k, size_t alpha, size_t A, int lda, size_t B, int ldb, size_t beta, size_t C, int ldc); cublasStatus_t cublasDgemm_v2( cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, int m, int n, int k, size_t alpha, size_t A, int lda, size_t B, int ldb, size_t beta, size_t C, int ldc); cublasStatus_t cublasSgemmEx( cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, int m, int n, int k, size_t alpha, size_t A, cublasDataType_t Atype, int lda, size_t B, cublasDataType_t Btype, int ldb, size_t beta, size_t C, cublasDataType_t Ctype, int ldc); cublasStatus_t cublasSetPointerMode_v2(cublasHandle_t handle, cublasPointerMode_t mode); """ # Parse global ffi ffi = cffi.FFI() ffi.cdef(src) # Load library for libnme in backends: try: lib = ffi.dlopen(libnme) break except OSError: pass else: ffi = None raise OSError("Could not load cublas library") global ERRORS for code, msg in ERRORS.items(): if code in CU.ERRORS: s = " | " + msg if s not in CU.ERRORS[code]: CU.ERRORS[code] += s else: CU.ERRORS[code] = msg