def remove_lib(self): # Unload the currently loaded dynamic linked library to be secure if self._lib is not None: _ctypes.FreeLibrary(self._lib._handle) if platform == 'win32' else _ctypes.dlclose(self._lib._handle) del self._lib self._lib = None # deactivate the cleanup finalizers for the current set of files if self._cleanup_files is not None: self._cleanup_files.detach() if self._cleanup_lib is not None: self._cleanup_lib.detach() # If file already exists, pull new names. This is necessary on a Windows machine, because # Python's ctype does not deal in any sort of manner well with dynamic linked libraries on this OS. if path.isfile(self.lib_file): [remove(s) for s in [self.src_file, self.lib_file, self.log_file]] if MPI: mpi_comm = MPI.COMM_WORLD mpi_rank = mpi_comm.Get_rank() basename = path.join(get_cache_dir(), self._cache_key) if mpi_rank == 0 else None basename = mpi_comm.bcast(basename, root=0) basename = basename + "_%d" % mpi_rank else: basename = path.join(get_cache_dir(), "%s_0" % self._cache_key) self.src_file = "%s.c" % basename self.lib_file = "%s.%s" % (basename, 'dll' if platform == 'win32' else 'so') self.log_file = "%s.log" % basename
def __init__(self, grid, ptype, pyfunc=None, funcname=None, funccode=None, py_ast=None, funcvars=None): self.grid = grid self.ptype = ptype # Derive meta information from pyfunc, if not given self.funcname = funcname or pyfunc.__name__ self.funcvars = funcvars or list(pyfunc.__code__.co_varnames) self.funccode = funccode or inspect.getsource(pyfunc.__code__) # Parse AST if it is not provided explicitly self.py_ast = py_ast or parse(fix_indentation(self.funccode)).body[0] if pyfunc is None: # Extract user context by inspecting the call stack stack = inspect.stack() try: user_ctx = stack[-1][0].f_globals user_ctx['math'] = globals()['math'] user_ctx['random'] = globals()['random'] except: print( "Warning: Could not access user context when merging kernels" ) user_ctx = globals() finally: del stack # Remove cyclic references # Compile and generate Python function from AST py_mod = Module(body=[self.py_ast]) exec(compile(py_mod, "<ast>", "exec"), user_ctx) self.pyfunc = user_ctx[self.funcname] else: self.pyfunc = pyfunc self.name = "%s%s" % (ptype.name, self.funcname) # Generate the kernel function and add the outer loop if self.ptype.uses_jit: kernelgen = KernelGenerator(grid, ptype) self.field_args = kernelgen.field_args kernel_ccode = kernelgen.generate(deepcopy(self.py_ast), self.funcvars) self.field_args = kernelgen.field_args loopgen = LoopGenerator(grid, ptype) adaptive = 'AdvectionRK45' in self.funcname self.ccode = loopgen.generate(self.funcname, self.field_args, kernel_ccode, adaptive=adaptive) basename = path.join(get_cache_dir(), self._cache_key) self.src_file = "%s.c" % basename self.lib_file = "%s.so" % basename self.log_file = "%s.log" % basename self._lib = None
class Random(object): stmt_import = """#include "parcels.h"\n\n""" fnct_seed = """ extern void pcls_seed(int seed){ parcels_seed(seed); } """ fnct_random = """ extern float pcls_random(){ return parcels_random(); } """ fnct_uniform = """ extern float pcls_uniform(float low, float high){ return parcels_uniform(low, high); } """ fnct_randint = """ extern int pcls_randint(int low, int high){ return parcels_randint(low, high); } """ fnct_normalvariate = """ extern float pcls_normalvariate(float loc, float scale){ return parcels_normalvariate(loc, scale); } """ ccode = stmt_import + fnct_seed ccode += fnct_random + fnct_uniform + fnct_randint + fnct_normalvariate src_file = path.join(get_cache_dir(), "random.c") lib_file = path.join(get_cache_dir(), "random.so") log_file = path.join(get_cache_dir(), "random.log") def __init__(self): self._lib = None @property def lib(self, compiler=GNUCompiler()): if self._lib is None: with open(self.src_file, 'w') as f: f.write(self.ccode) compiler.compile(self.src_file, self.lib_file, self.log_file) print("Compiled %s ==> %s" % ("random", self.lib_file)) self._lib = npct.load_library(self.lib_file, '.') return self._lib
class RandomC(object): stmt_import = """#include "parcels.h"\n\n""" fnct_seed = """ extern void pcls_seed(int seed){ parcels_seed(seed); } """ fnct_random = """ extern float pcls_random(){ return parcels_random(); } """ fnct_uniform = """ extern float pcls_uniform(float low, float high){ return parcels_uniform(low, high); } """ fnct_randint = """ extern int pcls_randint(int low, int high){ return parcels_randint(low, high); } """ fnct_normalvariate = """ extern float pcls_normalvariate(float loc, float scale){ return parcels_normalvariate(loc, scale); } """ fnct_expovariate = """ extern float pcls_expovariate(float lamb){ return parcels_expovariate(lamb); } """ fnct_vonmisesvariate = """ extern float pcls_vonmisesvariate(float mu, float kappa){ return parcels_vonmisesvariate(mu, kappa); } """ ccode = stmt_import + fnct_seed ccode += fnct_random + fnct_uniform + fnct_randint + fnct_normalvariate + fnct_expovariate + fnct_vonmisesvariate basename = path.join(get_cache_dir(), 'parcels_random_%s' % uuid.uuid4()) src_file = "%s.c" % basename lib_file = "%s.so" % basename log_file = "%s.log" % basename def __init__(self): self._lib = None @property def lib(self, compiler=GNUCompiler()): if self._lib is None: with open(self.src_file, 'w') as f: f.write(self.ccode) compiler.compile(self.src_file, self.lib_file, self.log_file) logger.info("Compiled %s ==> %s" % ("ParcelsRandom", self.lib_file)) self._lib = npct.load_library(self.lib_file, '.') return self._lib
def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, funccode=None, py_ast=None, funcvars=None): self.fieldset = fieldset self.ptype = ptype # Derive meta information from pyfunc, if not given self.funcname = funcname or pyfunc.__name__ if pyfunc is AdvectionRK4_3D: logger.info('Note that positive vertical velocity is assumed DOWNWARD by AdvectionRK4_3D') if funcvars is not None: self.funcvars = funcvars elif hasattr(pyfunc, '__code__'): self.funcvars = list(pyfunc.__code__.co_varnames) else: self.funcvars = None self.funccode = funccode or inspect.getsource(pyfunc.__code__) # Parse AST if it is not provided explicitly self.py_ast = py_ast or parse(fix_indentation(self.funccode)).body[0] if pyfunc is None: # Extract user context by inspecting the call stack stack = inspect.stack() try: user_ctx = stack[-1][0].f_globals user_ctx['math'] = globals()['math'] user_ctx['random'] = globals()['random'] user_ctx['ErrorCode'] = globals()['ErrorCode'] except: logger.warning("Could not access user context when merging kernels") user_ctx = globals() finally: del stack # Remove cyclic references # Compile and generate Python function from AST py_mod = Module(body=[self.py_ast]) exec(compile(py_mod, "<ast>", "exec"), user_ctx) self.pyfunc = user_ctx[self.funcname] else: self.pyfunc = pyfunc self.name = "%s%s" % (ptype.name, self.funcname) # Generate the kernel function and add the outer loop if self.ptype.uses_jit: kernelgen = KernelGenerator(fieldset, ptype) self.field_args = kernelgen.field_args kernel_ccode = kernelgen.generate(deepcopy(self.py_ast), self.funcvars) self.field_args = kernelgen.field_args self.const_args = kernelgen.const_args loopgen = LoopGenerator(fieldset, ptype) self.ccode = loopgen.generate(self.funcname, self.field_args, self.const_args, kernel_ccode) basename = path.join(get_cache_dir(), self._cache_key) self.src_file = "%s.c" % basename self.lib_file = "%s.%s" % (basename, 'dll' if platform == 'win32' else 'so') self.log_file = "%s.log" % basename self._lib = None
def remove_lib(self): # Unload the currently loaded dynamic linked library to be secure if self._lib is not None: _ctypes.FreeLibrary(self._lib._handle) if platform == 'win32' else _ctypes.dlclose(self._lib._handle) del self._lib self._lib = None # If file already exists, pull new names. This is necessary on a Windows machine, because # Python's ctype does not deal in any sort of manner well with dynamic linked libraries on this OS. if path.isfile(self.lib_file): map(remove, [self.src_file, self.lib_file, self.log_file]) basename = path.join(get_cache_dir(), self._cache_key) self.src_file = "%s.c" % basename self.lib_file = "%s.%s" % (basename, 'dll' if platform == 'win32' else 'so') self.log_file = "%s.log" % basename
def __init__(self, grid, ptype, pyfunc=None, funcname=None, funccode=None, py_ast=None, funcvars=None): self.grid = grid self.ptype = ptype # Derive meta information from pyfunc, if not given self.funcname = funcname or pyfunc.__name__ self.funcvars = funcvars or list(pyfunc.__code__.co_varnames) self.funccode = funccode or inspect.getsource(pyfunc.__code__) # Parse AST if it is not provided explicitly self.py_ast = py_ast or parse(fix_indentation(self.funccode)).body[0] if pyfunc is None: # Extract user context by inspecting the call stack stack = inspect.stack() try: user_ctx = stack[-1][0].f_globals user_ctx['math'] = globals()['math'] user_ctx['random'] = globals()['random'] user_ctx['ErrorCode'] = globals()['ErrorCode'] except: print("Warning: Could not access user context when merging kernels") user_ctx = globals() finally: del stack # Remove cyclic references # Compile and generate Python function from AST py_mod = Module(body=[self.py_ast]) exec(compile(py_mod, "<ast>", "exec"), user_ctx) self.pyfunc = user_ctx[self.funcname] else: self.pyfunc = pyfunc self.name = "%s%s" % (ptype.name, self.funcname) # Generate the kernel function and add the outer loop if self.ptype.uses_jit: kernelgen = KernelGenerator(grid, ptype) self.field_args = kernelgen.field_args kernel_ccode = kernelgen.generate(deepcopy(self.py_ast), self.funcvars) self.field_args = kernelgen.field_args loopgen = LoopGenerator(grid, ptype) self.ccode = loopgen.generate(self.funcname, self.field_args, kernel_ccode) basename = path.join(get_cache_dir(), self._cache_key) self.src_file = "%s.c" % basename self.lib_file = "%s.so" % basename self.log_file = "%s.log" % basename self._lib = None
def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, funccode=None, py_ast=None, funcvars=None, c_include="", delete_cfiles=True): self.fieldset = fieldset self.ptype = ptype self._lib = None self.delete_cfiles = delete_cfiles self._cleanup_files = None self._cleanup_lib = None # Derive meta information from pyfunc, if not given self.funcname = funcname or pyfunc.__name__ if pyfunc is AdvectionRK4_3D: warning = False if isinstance(fieldset.W, Field) and fieldset.W.creation_log != 'from_nemo' and \ fieldset.W._scaling_factor is not None and fieldset.W._scaling_factor > 0: warning = True if type(fieldset.W) in [SummedField, NestedField]: for f in fieldset.W: if f.creation_log != 'from_nemo' and f._scaling_factor is not None and f._scaling_factor > 0: warning = True if warning: logger.warning_once('Note that in AdvectionRK4_3D, vertical velocity is assumed positive towards increasing z.\n' ' If z increases downward and w is positive upward you can re-orient it downwards by setting fieldset.W.set_scaling_factor(-1.)') elif pyfunc is AdvectionAnalytical: if ptype.uses_jit: raise NotImplementedError('Analytical Advection only works in Scipy mode') if fieldset.U.interp_method != 'cgrid_velocity': raise NotImplementedError('Analytical Advection only works with C-grids') if fieldset.U.grid.gtype not in [GridCode.CurvilinearZGrid, GridCode.RectilinearZGrid]: raise NotImplementedError('Analytical Advection only works with Z-grids in the vertical') if funcvars is not None: self.funcvars = funcvars elif hasattr(pyfunc, '__code__'): self.funcvars = list(pyfunc.__code__.co_varnames) else: self.funcvars = None self.funccode = funccode or inspect.getsource(pyfunc.__code__) # Parse AST if it is not provided explicitly self.py_ast = py_ast or parse(fix_indentation(self.funccode)).body[0] if pyfunc is None: # Extract user context by inspecting the call stack stack = inspect.stack() try: user_ctx = stack[-1][0].f_globals user_ctx['math'] = globals()['math'] user_ctx['ParcelsRandom'] = globals()['ParcelsRandom'] user_ctx['random'] = globals()['random'] user_ctx['StateCode'] = globals()['StateCode'] user_ctx['OperationCode'] = globals()['OperationCode'] user_ctx['ErrorCode'] = globals()['ErrorCode'] except: logger.warning("Could not access user context when merging kernels") user_ctx = globals() finally: del stack # Remove cyclic references # Compile and generate Python function from AST py_mod = parse("") py_mod.body = [self.py_ast] exec(compile(py_mod, "<ast>", "exec"), user_ctx) self.pyfunc = user_ctx[self.funcname] else: self.pyfunc = pyfunc if version_info[0] < 3: numkernelargs = len(inspect.getargspec(self.pyfunc).args) else: numkernelargs = len(inspect.getfullargspec(self.pyfunc).args) assert numkernelargs == 3, \ 'Since Parcels v2.0, kernels do only take 3 arguments: particle, fieldset, time !! AND !! Argument order in field interpolation is time, depth, lat, lon.' self.name = "%s%s" % (ptype.name, self.funcname) # Generate the kernel function and add the outer loop if self.ptype.uses_jit: kernelgen = KernelGenerator(fieldset, ptype) kernel_ccode = kernelgen.generate(deepcopy(self.py_ast), self.funcvars) self.field_args = kernelgen.field_args self.vector_field_args = kernelgen.vector_field_args fieldset = self.fieldset for f in self.vector_field_args.values(): Wname = f.W.ccode_name if f.W else 'not_defined' for sF_name, sF_component in zip([f.U.ccode_name, f.V.ccode_name, Wname], ['U', 'V', 'W']): if sF_name not in self.field_args: if sF_name != 'not_defined': self.field_args[sF_name] = getattr(f, sF_component) self.const_args = kernelgen.const_args loopgen = LoopGenerator(fieldset, ptype) if path.isfile(c_include): with open(c_include, 'r') as f: c_include_str = f.read() else: c_include_str = c_include self.ccode = loopgen.generate(self.funcname, self.field_args, self.const_args, kernel_ccode, c_include_str) if MPI: mpi_comm = MPI.COMM_WORLD mpi_rank = mpi_comm.Get_rank() basename = path.join(get_cache_dir(), self._cache_key) if mpi_rank == 0 else None basename = mpi_comm.bcast(basename, root=0) basename = basename + "_%d" % mpi_rank else: basename = path.join(get_cache_dir(), "%s_0" % self._cache_key) self.src_file = "%s.c" % basename self.lib_file = "%s.%s" % (basename, 'dll' if platform == 'win32' else 'so') self.log_file = "%s.log" % basename
def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, funccode=None, py_ast=None, funcvars=None, c_include=""): self.fieldset = fieldset self.ptype = ptype self._lib = None # Derive meta information from pyfunc, if not given self.funcname = funcname or pyfunc.__name__ if pyfunc is AdvectionRK4_3D: warning = False if isinstance(fieldset.W, Field) and fieldset.W.creation_log != 'from_nemo' and \ fieldset.W._scaling_factor is not None and fieldset.W._scaling_factor > 0: warning = True if type(fieldset.W) in [SummedField, NestedField]: for f in fieldset.W: if f.creation_log != 'from_nemo' and f._scaling_factor is not None and f._scaling_factor > 0: warning = True if warning: logger.warning_once( 'Note that in AdvectionRK4_3D, vertical velocity is assumed positive towards increasing z.\n' ' If z increases downward and w is positive upward you can re-orient it downwards by setting fieldset.W.set_scaling_factor(-1.)' ) if funcvars is not None: self.funcvars = funcvars elif hasattr(pyfunc, '__code__'): self.funcvars = list(pyfunc.__code__.co_varnames) else: self.funcvars = None self.funccode = funccode or inspect.getsource(pyfunc.__code__) # Parse AST if it is not provided explicitly self.py_ast = py_ast or parse(fix_indentation(self.funccode)).body[0] if pyfunc is None: # Extract user context by inspecting the call stack stack = inspect.stack() try: user_ctx = stack[-1][0].f_globals user_ctx['math'] = globals()['math'] user_ctx['random'] = globals()['random'] user_ctx['ErrorCode'] = globals()['ErrorCode'] except: logger.warning( "Could not access user context when merging kernels") user_ctx = globals() finally: del stack # Remove cyclic references # Compile and generate Python function from AST py_mod = Module(body=[self.py_ast]) exec(compile(py_mod, "<ast>", "exec"), user_ctx) self.pyfunc = user_ctx[self.funcname] else: self.pyfunc = pyfunc assert len(inspect.getargspec(self.pyfunc).args) == 3, \ 'Since Parcels v2.0, kernels do only take 3 arguments: particle, fieldset, time !! AND !! Argument order in field interpolation is time, depth, lat, lon.' self.name = "%s%s" % (ptype.name, self.funcname) # Generate the kernel function and add the outer loop if self.ptype.uses_jit: kernelgen = KernelGenerator(fieldset, ptype) kernel_ccode = kernelgen.generate(deepcopy(self.py_ast), self.funcvars) self.field_args = kernelgen.field_args self.vector_field_args = kernelgen.vector_field_args fieldset = self.fieldset for fname in self.vector_field_args: f = getattr(fieldset, fname) Wname = f.W.name if f.W else 'not_defined' for sF in [f.U.name, f.V.name, Wname]: if sF not in self.field_args: try: self.field_args[sF] = getattr(fieldset, sF) except: continue self.const_args = kernelgen.const_args loopgen = LoopGenerator(fieldset, ptype) if path.isfile(c_include): with open(c_include, 'r') as f: c_include_str = f.read() else: c_include_str = c_include self.ccode = loopgen.generate(self.funcname, self.field_args, self.const_args, kernel_ccode, c_include_str) basename = path.join(get_cache_dir(), self._cache_key) self.src_file = "%s.c" % basename self.lib_file = "%s.%s" % (basename, 'dll' if platform == 'win32' else 'so') self.log_file = "%s.log" % basename