def test_get_host_cpu_features(self): try: features = llvm.get_host_cpu_features() except RuntimeError: # Allow non-x86 arch to pass even if an RuntimeError is raised triple = llvm.get_process_triple() # For now, we know for sure that x86 is supported. # We can restrict the test if we know this works on other arch. is_x86 = triple.startswith('x86') self.assertFalse(is_x86, msg="get_host_cpu_features() should not raise") return # Check the content of `features` self.assertIsInstance(features, dict) self.assertIsInstance(features, llvm.FeatureMap) for k, v in features.items(): self.assertIsInstance(k, str) self.assertTrue(k) # feature string cannot be empty self.assertIsInstance(v, bool) self.assertIsInstance(features.flatten(), str) re_term = r"[+\-][a-zA-Z0-9\._-]+" regex = r"^({0}|{0}(,{0})*)?$".format(re_term) # quick check for our regex self.assertIsNotNone(re.match(regex, "")) self.assertIsNotNone(re.match(regex, "+aa")) self.assertIsNotNone(re.match(regex, "+a,-bb")) # check CpuFeature.flatten() self.assertIsNotNone(re.match(regex, features.flatten()))
def _cpu_jit_constructor(): _binding_initialize() # PassManagerBuilder can be shared __pass_manager_builder = binding.PassManagerBuilder() __pass_manager_builder.inlining_threshold = 99999 # Inline all function calls __pass_manager_builder.loop_vectorize = True __pass_manager_builder.slp_vectorize = True __pass_manager_builder.opt_level = 3 # Most aggressive optimizations __cpu_features = binding.get_host_cpu_features().flatten() __cpu_name = binding.get_host_cpu_name() # Create compilation target, use default triple __cpu_target = binding.Target.from_default_triple() __cpu_target_machine = __cpu_target.create_target_machine( cpu=__cpu_name, features=__cpu_features, opt=3) __cpu_pass_manager = binding.ModulePassManager() __cpu_target_machine.add_analysis_passes(__cpu_pass_manager) __pass_manager_builder.populate(__cpu_pass_manager) # And an execution engine with a builtins backing module builtins_module = _generate_cpu_builtins_module(_float_ty) if "llvm" in debug_env: with open(builtins_module.name + '.parse.ll', 'w') as dump_file: dump_file.write(str(builtins_module)) __backing_mod = binding.parse_assembly(str(builtins_module)) __cpu_jit_engine = binding.create_mcjit_compiler(__backing_mod, __cpu_target_machine) return __cpu_jit_engine, __cpu_pass_manager, __cpu_target_machine
def _cpu_jit_constructor(): _binding_initialize() # PassManagerBuilder can be shared __pass_manager_builder = binding.PassManagerBuilder() __pass_manager_builder.loop_vectorize = True __pass_manager_builder.slp_vectorize = True __pass_manager_builder.opt_level = 2 __cpu_features = binding.get_host_cpu_features().flatten() __cpu_name = binding.get_host_cpu_name() # Create compilation target, use default triple __cpu_target = binding.Target.from_default_triple() # FIXME: reloc='static' is needed to avoid crashes on win64 # see: https://github.com/numba/llvmlite/issues/457 __cpu_target_machine = __cpu_target.create_target_machine( cpu=__cpu_name, features=__cpu_features, opt=2, reloc='static') __cpu_pass_manager = binding.ModulePassManager() __cpu_target_machine.add_analysis_passes(__cpu_pass_manager) __pass_manager_builder.populate(__cpu_pass_manager) # And an execution engine with a builtins backing module builtins_module = _generate_cpu_builtins_module( LLVMBuilderContext.float_ty) if "llvm" in debug_env: with open(builtins_module.name + '.parse.ll', 'w') as dump_file: dump_file.write(str(builtins_module)) __backing_mod = binding.parse_assembly(str(builtins_module)) __cpu_jit_engine = binding.create_mcjit_compiler(__backing_mod, __cpu_target_machine) return __cpu_jit_engine, __cpu_pass_manager, __cpu_target_machine
def get_target_machine(self): """Instantiate and return target machine.""" if not hasattr(self, '_llvm_module'): features = llvm.get_host_cpu_features().flatten() cpu = llvm.get_host_cpu_name() self._tm = llvm.Target.from_default_triple().create_target_machine( cpu=cpu, features=features, opt=1) return self._tm
def host(cls, name='host_cpu', strict=False, use_tracing_allocator=False): """Return target info for host CPU. """ key = (name, strict, use_tracing_allocator) target_info = TargetInfo._host_target_info_cache.get(key) if target_info is not None: return target_info import llvmlite.binding as ll target_info = cls(name=name, strict=strict, use_tracing_allocator=use_tracing_allocator) target_info.set('name', ll.get_host_cpu_name()) target_info.set('triple', ll.get_default_triple()) features = ','.join( ['-+'[int(v)] + k for k, v in ll.get_host_cpu_features().items()]) target_info.set('features', features) for tname, ctype in dict(bool=ctypes.c_bool, size_t=ctypes.c_size_t, ssize_t=ctypes.c_ssize_t, char=ctypes.c_char, uchar=ctypes.c_char, schar=ctypes.c_char, byte=ctypes.c_byte, ubyte=ctypes.c_ubyte, wchar=ctypes.c_wchar, short=ctypes.c_short, ushort=ctypes.c_ushort, int=ctypes.c_int, uint=ctypes.c_uint, long=ctypes.c_long, ulong=ctypes.c_ulong, longlong=ctypes.c_longlong, ulonglong=ctypes.c_ulonglong, float=ctypes.c_float, double=ctypes.c_double, longdouble=ctypes.c_longdouble, voidptr=ctypes.c_void_p).items(): target_info.type_sizeof[tname] = ctypes.sizeof(ctype) target_info.add_library('m') target_info.add_library('stdio') target_info.add_library('stdlib') target_info.add_library('rbclib') if use_tracing_allocator: target_info.set('fn_allocate_varlen_buffer', 'rbclib_tracing_allocate_varlen_buffer') target_info.set('fn_free_buffer', 'rbclib_tracing_free_buffer') else: target_info.set('fn_allocate_varlen_buffer', 'rbclib_allocate_varlen_buffer') target_info.set('fn_free_buffer', 'rbclib_free_buffer') cls._host_target_info_cache[key] = target_info return target_info
def _customize_tm_features(self): # For JIT target, we will use LLVM to get the feature map features = ll.get_host_cpu_features() if not config.ENABLE_AVX: # Disable all features with name starting with 'avx' for k in features: if k.startswith('avx'): features[k] = False # Set feature attributes return features.flatten()
def _customize_tm_features(self): # For JIT target, we will use LLVM to get the feature map features = ll.get_host_cpu_features() # There are various performance issues with AVX and LLVM 3.5 # (list at http://llvm.org/bugs/buglist.cgi?quicksearch=avx). # For now we'd rather disable it, since it can pessimize the code. if not config.ENABLE_AVX: # Disable all feature with name starting with 'avx' for k in features: if k.startswith('avx'): features[k] = False # Set feature attributes return features.flatten()
def __init__(self): llvm.initialize() llvm.initialize_all_targets() llvm.initialize_native_target() llvm.initialize_native_asmprinter() self.module = None self._llvmmod = llvm.parse_assembly("") self.target = llvm.Target.from_default_triple() self.cpu = llvm.get_host_cpu_name() self.cpu_features = llvm.get_host_cpu_features() self.target_machine = self.target.create_target_machine( cpu=self.cpu, features=self.cpu_features.flatten(), opt=2) llvm.check_jit_execution() self.ee = llvm.create_mcjit_compiler(self.llvmmod, self.target_machine) self.ee.finalize_object() self.fptr = None
def test_get_host_cpu_features(self): features = llvm.get_host_cpu_features() self.assertIsInstance(features, dict) self.assertIsInstance(features, llvm.FeatureMap) for k, v in features.items(): self.assertIsInstance(k, str) self.assertTrue(k) # feature string cannot be empty self.assertIsInstance(v, bool) self.assertIsInstance(features.flatten(), str) re_term = r"[+\-][a-zA-Z0-9\._-]+" regex = r"^({0}|{0}(,{0})*)?$".format(re_term) # quick check for our regex self.assertIsNotNone(re.match(regex, "")) self.assertIsNotNone(re.match(regex, "+aa")) self.assertIsNotNone(re.match(regex, "+a,-bb")) # check CpuFeature.flatten() self.assertIsNotNone(re.match(regex, features.flatten()))
def get_host_cpu_features(): """Get host CPU features using LLVM. The features may be modified due to user setting. See numba.config.ENABLE_AVX. """ try: features = ll.get_host_cpu_features() except RuntimeError: return '' else: if not config.ENABLE_AVX: # Disable all features with name starting with 'avx' for k in features: if k.startswith('avx'): features[k] = False # Set feature attributes return features.flatten()
def host(cls, name='host_cpu', strict=False): """Return target info for host CPU. """ key = (name, strict) target_info = cls._host_target_info_cache.get(key) if target_info is not None: return target_info import llvmlite.binding as ll target_info = TargetInfo(name=name, strict=strict) target_info.set('name', ll.get_host_cpu_name()) target_info.set('triple', ll.get_default_triple()) features = ','.join(['-+'[int(v)] + k for k, v in ll.get_host_cpu_features().items()]) target_info.set('features', features) for tname, ctype in dict( bool=ctypes.c_bool, size_t=ctypes.c_size_t, ssize_t=ctypes.c_ssize_t, char=ctypes.c_char, uchar=ctypes.c_char, schar=ctypes.c_char, byte=ctypes.c_byte, ubyte=ctypes.c_ubyte, wchar=ctypes.c_wchar, short=ctypes.c_short, ushort=ctypes.c_ushort, int=ctypes.c_int, uint=ctypes.c_uint, long=ctypes.c_long, ulong=ctypes.c_ulong, longlong=ctypes.c_longlong, ulonglong=ctypes.c_ulonglong, float=ctypes.c_float, double=ctypes.c_double, longdouble=ctypes.c_longdouble, ).items(): target_info.type_sizeof[tname] = ctypes.sizeof(ctype) cls._host_target_info_cache[key] = target_info return target_info
def create_execution_engine(self): """ Create an ExecutionEngine suitable for JIT code generation on the host CPU. The engine is reusable for an arbitrary number of modules. """ # Create a target machine representing the host target = binding.Target.from_default_triple() self.triple = target.triple # Keep the triple for later # Passing cpu, freatures and opt has not proved to be faster, but do it # anyway, just to show it. cpu = binding.get_host_cpu_name() features = binding.get_host_cpu_features() target_machine = target.create_target_machine( cpu=cpu, features=features.flatten(), opt=3, ) # And an execution engine with an empty backing module backing_mod = binding.parse_assembly("") engine = binding.create_mcjit_compiler(backing_mod, target_machine) return engine
def get_sys_info(): # delay these imports until now as they are only needed in this # function which then exits. import platform import json from numba import cuda as cu from numba.cuda import cudadrv from numba.cuda.cudadrv.driver import driver as cudriver import textwrap as tw import ctypes as ct import llvmlite.binding as llvmbind import locale from datetime import datetime from itertools import chain from subprocess import check_output, CalledProcessError try: fmt = "%-21s : %-s" print("-" * 80) print("__Time Stamp__") print(datetime.utcnow()) print("") print("__Hardware Information__") print(fmt % ("Machine", platform.machine())) print(fmt % ("CPU Name", llvmbind.get_host_cpu_name())) try: featuremap = llvmbind.get_host_cpu_features() except RuntimeError: print(fmt % ("CPU Features", "NA")) else: features = sorted( [key for key, value in featuremap.items() if value]) cpu_feat = tw.fill(' '.join(features), 80) print(fmt % ("CPU Features", "")) print(cpu_feat) print("") print("__OS Information__") print(fmt % ("Platform", platform.platform(aliased=True))) print(fmt % ("Release", platform.release())) system_name = platform.system() print(fmt % ("System Name", system_name)) print(fmt % ("Version", platform.version())) try: if system_name == 'Linux': info = platform.linux_distribution() elif system_name == 'Windows': info = platform.win32_ver() elif system_name == 'Darwin': info = platform.mac_ver() else: raise RuntimeError("Unknown system.") buf = ''.join([ x if x != '' else ' ' for x in list(chain.from_iterable(info)) ]) print(fmt % ("OS specific info", buf)) if system_name == 'Linux': print(fmt % ("glibc info", ' '.join(platform.libc_ver()))) except: print("Error: System name incorrectly identified or unknown.") print("") print("__Python Information__") print(fmt % ("Python Compiler", platform.python_compiler())) print(fmt % ("Python Implementation", platform.python_implementation())) print(fmt % ("Python Version", platform.python_version())) print(fmt % ("Python Locale ", ' '.join( [x for x in locale.getdefaultlocale() if x is not None]))) print("") print("__LLVM information__") print(fmt % ("LLVM version", '.'.join( [str(k) for k in llvmbind.llvm_version_info]))) print("") print("__CUDA Information__") # Look for GPUs try: cu.list_devices()[0] # will a device initialise? except BaseException as e: msg_not_found = "CUDA driver library cannot be found" msg_disabled_by_user = "******" msg_end = " or no CUDA enabled devices are present." msg_generic_problem = "Error: CUDA device intialisation problem." msg = getattr(e, 'msg', None) if msg is not None: if msg_not_found in msg: err_msg = msg_not_found + msg_end elif msg_disabled_by_user in msg: err_msg = msg_disabled_by_user + msg_end else: err_msg = msg_generic_problem + " Message:" + msg else: err_msg = msg_generic_problem + " " + str(e) # Best effort error report print("%s\nError class: %s" % (err_msg, str(type(e)))) else: try: cu.detect() dv = ct.c_int(0) cudriver.cuDriverGetVersion(ct.byref(dv)) print(fmt % ("CUDA driver version", dv.value)) print("CUDA libraries:") cudadrv.libs.test(sys.platform, print_paths=False) except: print( "Error: Probing CUDA failed (device and driver present, runtime problem?)\n" ) # Look for conda and conda information print("") print("__Conda Information__") cmd = ["conda", "info", "--json"] try: conda_out = check_output(cmd) except Exception as e: print("Conda not present/not working.\nError was %s\n" % e) else: data = ''.join(conda_out.decode("utf-8").splitlines()) jsond = json.loads(data) keys = [ 'conda_build_version', 'conda_env_version', 'platform', 'python_version', 'root_writable' ] for k in keys: try: print(fmt % (k, jsond[k])) except KeyError: pass # get info about current environment cmd = ["conda", "list"] try: conda_out = check_output(cmd) except CalledProcessError as e: print("Error: Conda command failed. Error was %s\n" % e.output) else: print("") print("__Current Conda Env__") data = conda_out.decode("utf-8").splitlines() for k in data: if k[0] != '#': # don't show where the env is, personal data print(k) print("-" * 80) except Exception as e: print("Error: The system reporting tool has failed unexpectedly.") print("Exception was:") print(e) finally: print( "%s" % "If requested, please copy and paste the information between\n" "the dashed (----) lines, or from a given specific section as\n" "appropriate.\n\n" "=============================================================\n" "IMPORTANT: Please ensure that you are happy with sharing the\n" "contents of the information present, any information that you\n" "wish to keep private you should remove before sharing.\n" "=============================================================\n")
def get_sysinfo(): # Gather the information that shouldn't raise exceptions sys_info = { _start: datetime.now(), _start_utc: datetime.utcnow(), _machine: platform.machine(), _cpu_name: llvmbind.get_host_cpu_name(), _cpu_count: multiprocessing.cpu_count(), _platform_name: platform.platform(aliased=True), _platform_release: platform.release(), _os_name: platform.system(), _os_version: platform.version(), _python_comp: platform.python_compiler(), _python_impl: platform.python_implementation(), _python_version: platform.python_version(), _numba_env_vars: {k: v for (k, v) in os.environ.items() if k.startswith('NUMBA_')}, _numba_version: version_number, _llvm_version: '.'.join(str(i) for i in llvmbind.llvm_version_info), _llvmlite_version: llvmlite_version, _roc_available: roc.is_available(), _psutil: _psutil_import, } # CPU features try: feature_map = llvmbind.get_host_cpu_features() except RuntimeError as e: _error_log.append(f'Error (CPU features): {e}') else: features = sorted([key for key, value in feature_map.items() if value]) sys_info[_cpu_features] = ' '.join(features) # Python locale # On MacOSX, getdefaultlocale can raise. Check again if Py > 3.7.5 try: # If $LANG is unset, getdefaultlocale() can return (None, None), make # sure we can encode this as strings by casting explicitly. sys_info[_python_locale] = '.'.join([str(i) for i in locale.getdefaultlocale()]) except Exception as e: _error_log.append(f'Error (locale): {e}') # CUDA information try: cu.list_devices()[0] # will a device initialise? except Exception as e: sys_info[_cu_dev_init] = False msg_not_found = "CUDA driver library cannot be found" msg_disabled_by_user = "******" msg_end = " or no CUDA enabled devices are present." msg_generic_problem = "CUDA device intialisation problem." msg = getattr(e, 'msg', None) if msg is not None: if msg_not_found in msg: err_msg = msg_not_found + msg_end elif msg_disabled_by_user in msg: err_msg = msg_disabled_by_user + msg_end else: err_msg = msg_generic_problem + " Message:" + msg else: err_msg = msg_generic_problem + " " + str(e) # Best effort error report _warning_log.append("Warning (cuda): %s\nException class: %s" % (err_msg, str(type(e)))) else: try: sys_info[_cu_dev_init] = True output = StringIO() with redirect_stdout(output): cu.detect() sys_info[_cu_detect_out] = output.getvalue() output.close() dv = ctypes.c_int(0) cudriver.cuDriverGetVersion(ctypes.byref(dv)) sys_info[_cu_drv_ver] = dv.value rtver = ctypes.c_int(0) curuntime.cudaRuntimeGetVersion(ctypes.byref(rtver)) sys_info[_cu_rt_ver] = rtver.value output = StringIO() with redirect_stdout(output): cudadrv.libs.test(sys.platform, print_paths=False) sys_info[_cu_lib_test] = output.getvalue() output.close() except Exception as e: _warning_log.append( "Warning (cuda): Probing CUDA failed " "(device and driver present, runtime problem?)\n" f"(cuda) {type(e)}: {e}") # ROC information # If no ROC try and report why if not sys_info[_roc_available]: from numba.roc.hsadrv.driver import hsa try: hsa.is_available except Exception as e: msg = str(e) else: msg = 'No ROC toolchains found.' _warning_log.append(f"Warning (roc): Error initialising ROC: {msg}") toolchains = [] try: libhlc.HLC() toolchains.append('librocmlite library') except Exception: pass try: cmd = hlc.CmdLine().check_tooling() toolchains.append('ROC command line tools') except Exception: pass sys_info[_roc_toolchains] = toolchains try: # ROC might not be available due to lack of tool chain, but HSA # agents may be listed from numba.roc.hsadrv.driver import hsa, dgpu_count def decode(x): return x.decode('utf-8') if isinstance(x, bytes) else x sys_info[_hsa_agents_count] = len(hsa.agents) agents = [] for i, agent in enumerate(hsa.agents): agents.append({ 'Agent id': i, 'Vendor': decode(agent.vendor_name), 'Name': decode(agent.name), 'Type': agent.device, }) sys_info[_hsa_agents] = agents _dgpus = [] for a in hsa.agents: if a.is_component and a.device == 'GPU': _dgpus.append(decode(a.name)) sys_info[_hsa_gpus_count] = dgpu_count() sys_info[_hsa_gpus] = ', '.join(_dgpus) except Exception as e: _warning_log.append( "Warning (roc): No HSA Agents found, " f"encountered exception when searching: {e}") # SVML information # Replicate some SVML detection logic from numba.__init__ here. # If SVML load fails in numba.__init__ the splitting of the logic # here will help diagnosing the underlying issue. svml_lib_loaded = True try: if sys.platform.startswith('linux'): llvmbind.load_library_permanently("libsvml.so") elif sys.platform.startswith('darwin'): llvmbind.load_library_permanently("libsvml.dylib") elif sys.platform.startswith('win'): llvmbind.load_library_permanently("svml_dispmd") else: svml_lib_loaded = False except Exception: svml_lib_loaded = False func = getattr(llvmbind.targets, "has_svml", None) sys_info[_llvm_svml_patched] = func() if func else False sys_info[_svml_state] = config.USING_SVML sys_info[_svml_loaded] = svml_lib_loaded sys_info[_svml_operational] = all(( sys_info[_svml_state], sys_info[_svml_loaded], sys_info[_llvm_svml_patched], )) # Check which threading backends are available. def parse_error(e, backend): # parses a linux based error message, this is to provide feedback # and hide user paths etc try: path, problem, symbol = [x.strip() for x in e.msg.split(':')] extn_dso = os.path.split(path)[1] if backend in extn_dso: return "%s: %s" % (problem, symbol) except Exception: pass return "Unknown import problem." try: from numba.np.ufunc import tbbpool # NOQA sys_info[_tbb_thread] = True except ImportError as e: # might be a missing symbol due to e.g. tbb libraries missing sys_info[_tbb_thread] = False sys_info[_tbb_error] = parse_error(e, 'tbbpool') try: from numba.np.ufunc import omppool sys_info[_openmp_thread] = True sys_info[_openmp_vendor] = omppool.openmp_vendor except ImportError as e: sys_info[_openmp_thread] = False sys_info[_openmp_error] = parse_error(e, 'omppool') try: from numba.np.ufunc import workqueue # NOQA sys_info[_wkq_thread] = True except ImportError as e: sys_info[_wkq_thread] = True sys_info[_wkq_error] = parse_error(e, 'workqueue') # Look for conda and installed packages information cmd = ('conda', 'info', '--json') try: conda_out = check_output(cmd) except Exception as e: _warning_log.append(f'Warning: Conda not available.\n Error was {e}\n') # Conda is not available, try pip list to list installed packages cmd = (sys.executable, '-m', 'pip', 'list') try: reqs = check_output(cmd) except Exception as e: _error_log.append(f'Error (pip): {e}') else: sys_info[_inst_pkg] = reqs.decode().splitlines() else: jsond = json.loads(conda_out.decode()) keys = { 'conda_build_version': _conda_build_ver, 'conda_env_version': _conda_env_ver, 'platform': _conda_platform, 'python_version': _conda_python_ver, 'root_writable': _conda_root_writable, } for conda_k, sysinfo_k in keys.items(): sys_info[sysinfo_k] = jsond.get(conda_k, 'N/A') # Get info about packages in current environment cmd = ('conda', 'list') try: conda_out = check_output(cmd) except CalledProcessError as e: _error_log.append(f'Error (conda): {e}') else: data = conda_out.decode().splitlines() sys_info[_inst_pkg] = [l for l in data if not l.startswith('#')] sys_info.update(get_os_spec_info(sys_info[_os_name])) sys_info[_errors] = _error_log sys_info[_warnings] = _warning_log sys_info[_runtime] = (datetime.now() - sys_info[_start]).total_seconds() return sys_info
def get_sys_info(): # delay these imports until now as they are only needed in this # function which then exits. import platform import json from numba import cuda as cu from numba.cuda import cudadrv from numba.cuda.cudadrv.driver import driver as cudriver import textwrap as tw import ctypes as ct import llvmlite.binding as llvmbind import locale from datetime import datetime from itertools import chain from subprocess import check_output, CalledProcessError try: fmt = "%-21s : %-s" print("-" * 80) print("__Time Stamp__") print(datetime.utcnow()) print("") print("__Hardware Information__") print(fmt % ("Machine", platform.machine())) print(fmt % ("CPU Name", llvmbind.get_host_cpu_name())) try: featuremap = llvmbind.get_host_cpu_features() except RuntimeError: print(fmt % ("CPU Features", "NA")) else: features = sorted([key for key, value in featuremap.items() if value]) cpu_feat = tw.fill(' '.join(features), 80) print(fmt % ("CPU Features", "")) print(cpu_feat) print("") print("__OS Information__") print(fmt % ("Platform", platform.platform(aliased=True))) print(fmt % ("Release", platform.release())) system_name = platform.system() print(fmt % ("System Name", system_name)) print(fmt % ("Version", platform.version())) try: if system_name == 'Linux': info = platform.linux_distribution() elif system_name == 'Windows': info = platform.win32_ver() elif system_name == 'Darwin': info = platform.mac_ver() else: raise RuntimeError("Unknown system.") buf = ''.join([x if x != '' else ' ' for x in list(chain.from_iterable(info))]) print(fmt % ("OS specific info", buf)) if system_name == 'Linux': print(fmt % ("glibc info", ' '.join(platform.libc_ver()))) except: print("Error: System name incorrectly identified or unknown.") print("") print("__Python Information__") print(fmt % ("Python Compiler", platform.python_compiler())) print( fmt % ("Python Implementation", platform.python_implementation())) print(fmt % ("Python Version", platform.python_version())) print( fmt % ("Python Locale ", ' '.join( [x for x in locale.getdefaultlocale() if x is not None]))) print("") print("__LLVM information__") print( fmt % ("LLVM version", '.'.join( [str(k) for k in llvmbind.llvm_version_info]))) print("") print("__CUDA Information__") # Look for GPUs try: cu.list_devices()[0] # will a device initialise? except BaseException as e: msg_not_found = "CUDA driver library cannot be found" msg_disabled_by_user = "******" msg_end = " or no CUDA enabled devices are present." msg_generic_problem = "Error: CUDA device intialisation problem." msg = getattr(e, 'msg', None) if msg is not None: if msg_not_found in msg: err_msg = msg_not_found + msg_end elif msg_disabled_by_user in msg: err_msg = msg_disabled_by_user + msg_end else: err_msg = msg_generic_problem + " Message:" + msg else: err_msg = msg_generic_problem + " " + str(e) # Best effort error report print("%s\nError class: %s" % (err_msg, str(type(e)))) else: try: cu.detect() dv = ct.c_int(0) cudriver.cuDriverGetVersion(ct.byref(dv)) print(fmt % ("CUDA driver version", dv.value)) print("CUDA libraries:") cudadrv.libs.test(sys.platform, print_paths=False) except: print( "Error: Probing CUDA failed (device and driver present, runtime problem?)\n") # Look for conda and conda information print("") print("__Conda Information__") cmd = ["conda", "info", "--json"] try: conda_out = check_output(cmd) except Exception as e: print( "Conda not present/not working.\nError was %s\n" % e) else: data = ''.join(conda_out.decode("utf-8").splitlines()) jsond = json.loads(data) keys = ['conda_build_version', 'conda_env_version', 'platform', 'python_version', 'root_writable'] for k in keys: try: print(fmt % (k, jsond[k])) except KeyError: pass # get info about current environment cmd = ["conda", "list"] try: conda_out = check_output(cmd) except CalledProcessError as e: print("Error: Conda command failed. Error was %s\n" % e.output) else: print("") print("__Current Conda Env__") data = conda_out.decode("utf-8").splitlines() for k in data: if k[0] != '#': # don't show where the env is, personal data print(k) print("-" * 80) except Exception as e: print("Error: The system reporting tool has failed unexpectedly.") print("Exception was:") print(e) finally: print( "%s" % "If requested, please copy and paste the information between\n" "the dashed (----) lines, or from a given specific section as\n" "appropriate.\n\n" "=============================================================\n" "IMPORTANT: Please ensure that you are happy with sharing the\n" "contents of the information present, any information that you\n" "wish to keep private you should remove before sharing.\n" "=============================================================\n")
def get_sysinfo(): # Gather the information that shouldn't raise exceptions sys_info = { _start: datetime.now(), _start_utc: datetime.utcnow(), _machine: platform.machine(), _cpu_name: llvmbind.get_host_cpu_name(), _cpu_count: multiprocessing.cpu_count(), _platform_name: platform.platform(aliased=True), _platform_release: platform.release(), _os_name: platform.system(), _os_version: platform.version(), _python_comp: platform.python_compiler(), _python_impl: platform.python_implementation(), _python_version: platform.python_version(), _numba_env_vars: {k: v for (k, v) in os.environ.items() if k.startswith('NUMBA_')}, _numba_version: version_number, _llvm_version: '.'.join(str(i) for i in llvmbind.llvm_version_info), _llvmlite_version: llvmlite_version, _psutil: _psutil_import, } # CPU features try: feature_map = llvmbind.get_host_cpu_features() except RuntimeError as e: _error_log.append(f'Error (CPU features): {e}') else: features = sorted([key for key, value in feature_map.items() if value]) sys_info[_cpu_features] = ' '.join(features) # Python locale # On MacOSX, getdefaultlocale can raise. Check again if Py > 3.7.5 try: # If $LANG is unset, getdefaultlocale() can return (None, None), make # sure we can encode this as strings by casting explicitly. sys_info[_python_locale] = '.'.join( [str(i) for i in locale.getdefaultlocale()]) except Exception as e: _error_log.append(f'Error (locale): {e}') # CUDA information try: cu.list_devices()[0] # will a device initialise? except Exception as e: sys_info[_cu_dev_init] = False msg_not_found = "CUDA driver library cannot be found" msg_disabled_by_user = "******" msg_end = " or no CUDA enabled devices are present." msg_generic_problem = "CUDA device initialisation problem." msg = getattr(e, 'msg', None) if msg is not None: if msg_not_found in msg: err_msg = msg_not_found + msg_end elif msg_disabled_by_user in msg: err_msg = msg_disabled_by_user + msg_end else: err_msg = msg_generic_problem + " Message:" + msg else: err_msg = msg_generic_problem + " " + str(e) # Best effort error report _warning_log.append("Warning (cuda): %s\nException class: %s" % (err_msg, str(type(e)))) else: try: sys_info[_cu_dev_init] = True output = StringIO() with redirect_stdout(output): cu.detect() sys_info[_cu_detect_out] = output.getvalue() output.close() sys_info[_cu_drv_ver] = '%s.%s' % cudriver.get_version() sys_info[_cu_rt_ver] = '%s.%s' % curuntime.get_version() output = StringIO() with redirect_stdout(output): cudadrv.libs.test(sys.platform, print_paths=False) sys_info[_cu_lib_test] = output.getvalue() output.close() try: from cuda import cuda # noqa: F401 nvidia_bindings_available = True except ImportError: nvidia_bindings_available = False sys_info[_cu_nvidia_bindings] = nvidia_bindings_available nv_binding_used = bool(cudadrv.driver.USE_NV_BINDING) sys_info[_cu_nvidia_bindings_used] = nv_binding_used except Exception as e: _warning_log.append( "Warning (cuda): Probing CUDA failed " "(device and driver present, runtime problem?)\n" f"(cuda) {type(e)}: {e}") # NumPy information sys_info[_numpy_version] = np.version.full_version try: # NOTE: These consts were added in NumPy 1.20 from numpy.core._multiarray_umath import ( __cpu_features__, __cpu_dispatch__, __cpu_baseline__, ) except ImportError: sys_info[_numpy_AVX512_SKX_detected] = False else: feat_filtered = [k for k, v in __cpu_features__.items() if v] sys_info[_numpy_supported_simd_features] = feat_filtered sys_info[_numpy_supported_simd_dispatch] = __cpu_dispatch__ sys_info[_numpy_supported_simd_baseline] = __cpu_baseline__ sys_info[_numpy_AVX512_SKX_detected] = \ __cpu_features__.get("AVX512_SKX", False) # SVML information # Replicate some SVML detection logic from numba.__init__ here. # If SVML load fails in numba.__init__ the splitting of the logic # here will help diagnosing the underlying issue. svml_lib_loaded = True try: if sys.platform.startswith('linux'): llvmbind.load_library_permanently("libsvml.so") elif sys.platform.startswith('darwin'): llvmbind.load_library_permanently("libsvml.dylib") elif sys.platform.startswith('win'): llvmbind.load_library_permanently("svml_dispmd") else: svml_lib_loaded = False except Exception: svml_lib_loaded = False func = getattr(llvmbind.targets, "has_svml", None) sys_info[_llvm_svml_patched] = func() if func else False sys_info[_svml_state] = config.USING_SVML sys_info[_svml_loaded] = svml_lib_loaded sys_info[_svml_operational] = all(( sys_info[_svml_state], sys_info[_svml_loaded], sys_info[_llvm_svml_patched], )) # Check which threading backends are available. def parse_error(e, backend): # parses a linux based error message, this is to provide feedback # and hide user paths etc try: path, problem, symbol = [x.strip() for x in e.msg.split(':')] extn_dso = os.path.split(path)[1] if backend in extn_dso: return "%s: %s" % (problem, symbol) except Exception: pass return "Unknown import problem." try: # check import is ok, this means the DSO linkage is working from numba.np.ufunc import tbbpool # NOQA # check that the version is compatible, this is a check performed at # runtime (well, compile time), it will also ImportError if there's # a problem. from numba.np.ufunc.parallel import _check_tbb_version_compatible _check_tbb_version_compatible() sys_info[_tbb_thread] = True except ImportError as e: # might be a missing symbol due to e.g. tbb libraries missing sys_info[_tbb_thread] = False sys_info[_tbb_error] = parse_error(e, 'tbbpool') try: from numba.np.ufunc import omppool sys_info[_openmp_thread] = True sys_info[_openmp_vendor] = omppool.openmp_vendor except ImportError as e: sys_info[_openmp_thread] = False sys_info[_openmp_error] = parse_error(e, 'omppool') try: from numba.np.ufunc import workqueue # NOQA sys_info[_wkq_thread] = True except ImportError as e: sys_info[_wkq_thread] = True sys_info[_wkq_error] = parse_error(e, 'workqueue') # Look for conda and installed packages information cmd = ('conda', 'info', '--json') try: conda_out = check_output(cmd) except Exception as e: _warning_log.append(f'Warning: Conda not available.\n Error was {e}\n') # Conda is not available, try pip list to list installed packages cmd = (sys.executable, '-m', 'pip', 'list') try: reqs = check_output(cmd) except Exception as e: _error_log.append(f'Error (pip): {e}') else: sys_info[_inst_pkg] = reqs.decode().splitlines() else: jsond = json.loads(conda_out.decode()) keys = { 'conda_build_version': _conda_build_ver, 'conda_env_version': _conda_env_ver, 'platform': _conda_platform, 'python_version': _conda_python_ver, 'root_writable': _conda_root_writable, } for conda_k, sysinfo_k in keys.items(): sys_info[sysinfo_k] = jsond.get(conda_k, 'N/A') # Get info about packages in current environment cmd = ('conda', 'list') try: conda_out = check_output(cmd) except CalledProcessError as e: _error_log.append(f'Error (conda): {e}') else: data = conda_out.decode().splitlines() sys_info[_inst_pkg] = [l for l in data if not l.startswith('#')] sys_info.update(get_os_spec_info(sys_info[_os_name])) sys_info[_errors] = _error_log sys_info[_warnings] = _warning_log sys_info[_runtime] = (datetime.now() - sys_info[_start]).total_seconds() return sys_info
def get_sys_info(): # delay these imports until now as they are only needed in this # function which then exits. import platform import json from numba import config from numba import cuda as cu from numba.cuda import cudadrv from numba.cuda.cudadrv.driver import driver as cudriver from numba import roc from numba.roc.hlc import hlc, libhlc import textwrap as tw import ctypes as ct import llvmlite.binding as llvmbind import locale from datetime import datetime from itertools import chain from subprocess import check_output, CalledProcessError try: fmt = "%-35s : %-s" print("-" * 80) print("__Time Stamp__") print(datetime.utcnow()) print("") print("__Hardware Information__") print(fmt % ("Machine", platform.machine())) print(fmt % ("CPU Name", llvmbind.get_host_cpu_name())) try: featuremap = llvmbind.get_host_cpu_features() except RuntimeError: print(fmt % ("CPU Features", "NA")) else: features = sorted([key for key, value in featuremap.items() if value]) cpu_feat = tw.fill(' '.join(features), 80) print(fmt % ("CPU Features", "")) print(cpu_feat) print("") print("__OS Information__") print(fmt % ("Platform", platform.platform(aliased=True))) print(fmt % ("Release", platform.release())) system_name = platform.system() print(fmt % ("System Name", system_name)) print(fmt % ("Version", platform.version())) try: if system_name == 'Linux': info = platform.linux_distribution() elif system_name == 'Windows': info = platform.win32_ver() elif system_name == 'Darwin': info = platform.mac_ver() else: raise RuntimeError("Unknown system.") buf = ''.join([x if x != '' else ' ' for x in list(chain.from_iterable(info))]) print(fmt % ("OS specific info", buf)) if system_name == 'Linux': print(fmt % ("glibc info", ' '.join(platform.libc_ver()))) except: print("Error: System name incorrectly identified or unknown.") print("") print("__Python Information__") print(fmt % ("Python Compiler", platform.python_compiler())) print( fmt % ("Python Implementation", platform.python_implementation())) print(fmt % ("Python Version", platform.python_version())) print( fmt % ("Python Locale ", ' '.join( [x for x in locale.getdefaultlocale() if x is not None]))) print("") print("__LLVM information__") print( fmt % ("LLVM version", '.'.join( [str(k) for k in llvmbind.llvm_version_info]))) print("") print("__CUDA Information__") # Look for GPUs try: cu.list_devices()[0] # will a device initialise? except BaseException as e: msg_not_found = "CUDA driver library cannot be found" msg_disabled_by_user = "******" msg_end = " or no CUDA enabled devices are present." msg_generic_problem = "Error: CUDA device intialisation problem." msg = getattr(e, 'msg', None) if msg is not None: if msg_not_found in msg: err_msg = msg_not_found + msg_end elif msg_disabled_by_user in msg: err_msg = msg_disabled_by_user + msg_end else: err_msg = msg_generic_problem + " Message:" + msg else: err_msg = msg_generic_problem + " " + str(e) # Best effort error report print("%s\nError class: %s" % (err_msg, str(type(e)))) else: try: cu.detect() dv = ct.c_int(0) cudriver.cuDriverGetVersion(ct.byref(dv)) print(fmt % ("CUDA driver version", dv.value)) print("CUDA libraries:") cudadrv.libs.test(sys.platform, print_paths=False) except: print( "Error: Probing CUDA failed (device and driver present, runtime problem?)\n") print("") print("__ROC Information__") roc_is_available = roc.is_available() print(fmt % ("ROC available", roc_is_available)) toolchains = [] try: libhlc.HLC() toolchains.append('librocmlite library') except: pass try: cmd = hlc.CmdLine().check_tooling() toolchains.append('ROC command line tools') except: pass # if no ROC try and report why if not roc_is_available: from numba.roc.hsadrv.driver import hsa try: hsa.is_available except BaseException as e: msg = str(e) else: msg = 'No ROC toolchains found.' print(fmt % ("Error initialising ROC due to", msg)) if toolchains: print(fmt % ("Available Toolchains", ', '.join(toolchains))) try: # ROC might not be available due to lack of tool chain, but HSA # agents may be listed from numba.roc.hsadrv.driver import hsa, dgpu_count decode = lambda x: x.decode('utf-8') if isinstance(x, bytes) else x print("\nFound %s HSA Agents:" % len(hsa.agents)) for i, agent in enumerate(hsa.agents): print('Agent id : %s' % i) print(' vendor: %s' % decode(agent.vendor_name)) print(' name: %s' % decode(agent.name)) print(' type: %s' % agent.device) print("") _dgpus = [] for a in hsa.agents: if a.is_component and a.device == 'GPU': _dgpus.append(decode(a.name)) print(fmt % ("Found %s discrete GPU(s)" % dgpu_count(), \ ', '.join(_dgpus))) except Exception as e: print("No HSA Agents found, encountered exception when searching:") print(e) print("") print("__SVML Information__") # replicate some SVML detection logic from numba.__init__ here. # if SVML load fails in numba.__init__ the splitting of the logic # here will help diagnosis of the underlying issue have_svml_library = True try: if sys.platform.startswith('linux'): llvmbind.load_library_permanently("libsvml.so") elif sys.platform.startswith('darwin'): llvmbind.load_library_permanently("libsvml.dylib") elif sys.platform.startswith('win'): llvmbind.load_library_permanently("svml_dispmd") else: have_svml_library = False except: have_svml_library = False func = getattr(llvmbind.targets, "has_svml", None) llvm_svml_patched = func() if func is not None else False svml_operational = (config.USING_SVML and llvm_svml_patched \ and have_svml_library) print(fmt % ("SVML state, config.USING_SVML", config.USING_SVML)) print(fmt % ("SVML library found and loaded", have_svml_library)) print(fmt % ("llvmlite using SVML patched LLVM", llvm_svml_patched)) print(fmt % ("SVML operational:", svml_operational)) # Look for conda and conda information print("") print("__Conda Information__") cmd = ["conda", "info", "--json"] try: conda_out = check_output(cmd) except Exception as e: print( "Conda not present/not working.\nError was %s\n" % e) else: data = ''.join(conda_out.decode("utf-8").splitlines()) jsond = json.loads(data) keys = ['conda_build_version', 'conda_env_version', 'platform', 'python_version', 'root_writable'] for k in keys: try: print(fmt % (k, jsond[k])) except KeyError: pass # get info about current environment cmd = ["conda", "list"] try: conda_out = check_output(cmd) except CalledProcessError as e: print("Error: Conda command failed. Error was %s\n" % e.output) else: print("") print("__Current Conda Env__") data = conda_out.decode("utf-8").splitlines() for k in data: if k[0] != '#': # don't show where the env is, personal data print(k) print("-" * 80) except Exception as e: print("Error: The system reporting tool has failed unexpectedly.") print("Exception was:") print(e) finally: print( "%s" % "If requested, please copy and paste the information between\n" "the dashed (----) lines, or from a given specific section as\n" "appropriate.\n\n" "=============================================================\n" "IMPORTANT: Please ensure that you are happy with sharing the\n" "contents of the information present, any information that you\n" "wish to keep private you should remove before sharing.\n" "=============================================================\n")
def get_sys_info(): # delay these imports until now as they are only needed in this # function which then exits. import platform import json import multiprocessing from numba import config from numba import cuda as cu from numba.cuda import cudadrv from numba.cuda.cudadrv.driver import driver as cudriver from numba import roc from numba.roc.hlc import hlc, libhlc import textwrap as tw import ctypes as ct import llvmlite.binding as llvmbind import locale from datetime import datetime from itertools import chain from subprocess import check_output, CalledProcessError try: fmt = "%-45s : %-s" print("-" * 80) print("__Time Stamp__") print(datetime.utcnow()) print("") print("__Hardware Information__") system_name = platform.system() print(fmt % ("Machine", platform.machine())) print(fmt % ("CPU Name", llvmbind.get_host_cpu_name())) if system_name == 'Linux': strmatch = 'Cpus_allowed' try: loc = '/proc/self/status' with open(loc, 'rt') as f: proc_stat = f.read().splitlines() for x in proc_stat: if x.startswith(strmatch): if x.startswith('%s:' % strmatch): hexnum = '0x%s' % x.split(':')[1].strip() acc_cpus = int(hexnum, 16) _n = str(bin(acc_cpus).count('1')) print(fmt % ("Number of accessible CPU cores", _n)) elif x.startswith('%s_list:' % strmatch): _a = x.split(':')[1].strip() print(fmt % ("Listed accessible CPUs cores", _a)) except Exception: print(fmt % ("CPU count", multiprocessing.cpu_count())) # See if CFS is in place # https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt try: def scrape_lines(loc): with open(loc, 'rt') as f: return f.read().splitlines() loc = '/sys/fs/cgroup/cpuacct/cpu.cfs_period_us' cfs_period = int(scrape_lines(loc)[0]) loc = '/sys/fs/cgroup/cpuacct/cpu.cfs_quota_us' cfs_quota = int(scrape_lines(loc)[0]) if cfs_quota == -1: print(fmt % ("CFS restrictions", "None")) else: runtime_amount = float(cfs_quota) / float(cfs_period) print(fmt % ("CFS restrictions (CPUs worth of runtime)", runtime_amount)) except Exception: print(fmt % ("CFS restrictions", 'Information not available')) else: print(fmt % ("CPU count", multiprocessing.cpu_count())) try: featuremap = llvmbind.get_host_cpu_features() except RuntimeError: print(fmt % ("CPU Features", "NA")) else: features = sorted( [key for key, value in featuremap.items() if value]) cpu_feat = tw.fill(' '.join(features), 80) print(fmt % ("CPU Features", "")) print(cpu_feat) print("") print("__OS Information__") print(fmt % ("Platform", platform.platform(aliased=True))) print(fmt % ("Release", platform.release())) print(fmt % ("System Name", system_name)) print(fmt % ("Version", platform.version())) try: if system_name == 'Linux': info = platform.linux_distribution() elif system_name == 'Windows': info = platform.win32_ver() elif system_name == 'Darwin': info = platform.mac_ver() else: raise RuntimeError("Unknown system.") buf = ''.join([ x if x != '' else ' ' for x in list(chain.from_iterable(info)) ]) print(fmt % ("OS specific info", buf)) if system_name == 'Linux': print(fmt % ("glibc info", ' '.join(platform.libc_ver()))) except: print("Error: System name incorrectly identified or unknown.") print("") print("__Python Information__") print(fmt % ("Python Compiler", platform.python_compiler())) print(fmt % ("Python Implementation", platform.python_implementation())) print(fmt % ("Python Version", platform.python_version())) lcl = [] try: for x in locale.getdefaultlocale(): if x is not None: lcl.append(x) except Exception as e: lcl.append(str(e)) print(fmt % ("Python Locale ", ' '.join(lcl))) print("") print("__LLVM information__") print(fmt % ("LLVM version", '.'.join( [str(k) for k in llvmbind.llvm_version_info]))) print("") print("__CUDA Information__") # Look for GPUs try: cu.list_devices()[0] # will a device initialise? except Exception as e: msg_not_found = "CUDA driver library cannot be found" msg_disabled_by_user = "******" msg_end = " or no CUDA enabled devices are present." msg_generic_problem = "Error: CUDA device intialisation problem." msg = getattr(e, 'msg', None) if msg is not None: if msg_not_found in msg: err_msg = msg_not_found + msg_end elif msg_disabled_by_user in msg: err_msg = msg_disabled_by_user + msg_end else: err_msg = msg_generic_problem + " Message:" + msg else: err_msg = msg_generic_problem + " " + str(e) # Best effort error report print("%s\nError class: %s" % (err_msg, str(type(e)))) else: try: cu.detect() dv = ct.c_int(0) cudriver.cuDriverGetVersion(ct.byref(dv)) print(fmt % ("CUDA driver version", dv.value)) print("CUDA libraries:") cudadrv.libs.test(sys.platform, print_paths=False) except: print( "Error: Probing CUDA failed (device and driver present, runtime problem?)\n" ) print("") print("__ROC Information__") roc_is_available = roc.is_available() print(fmt % ("ROC available", roc_is_available)) toolchains = [] try: libhlc.HLC() toolchains.append('librocmlite library') except: pass try: cmd = hlc.CmdLine().check_tooling() toolchains.append('ROC command line tools') except: pass # if no ROC try and report why if not roc_is_available: from numba.roc.hsadrv.driver import hsa try: hsa.is_available except Exception as e: msg = str(e) else: msg = 'No ROC toolchains found.' print(fmt % ("Error initialising ROC due to", msg)) if toolchains: print(fmt % ("Available Toolchains", ', '.join(toolchains))) try: # ROC might not be available due to lack of tool chain, but HSA # agents may be listed from numba.roc.hsadrv.driver import hsa, dgpu_count decode = lambda x: x.decode('utf-8') if isinstance(x, bytes) else x print("\nFound %s HSA Agents:" % len(hsa.agents)) for i, agent in enumerate(hsa.agents): print('Agent id : %s' % i) print(' vendor: %s' % decode(agent.vendor_name)) print(' name: %s' % decode(agent.name)) print(' type: %s' % agent.device) print("") _dgpus = [] for a in hsa.agents: if a.is_component and a.device == 'GPU': _dgpus.append(decode(a.name)) print(fmt % ("Found %s discrete GPU(s)" % dgpu_count(), \ ', '.join(_dgpus))) except Exception as e: print("No HSA Agents found, encountered exception when searching:") print(e) print("") print("__SVML Information__") # replicate some SVML detection logic from numba.__init__ here. # if SVML load fails in numba.__init__ the splitting of the logic # here will help diagnosis of the underlying issue have_svml_library = True try: if sys.platform.startswith('linux'): llvmbind.load_library_permanently("libsvml.so") elif sys.platform.startswith('darwin'): llvmbind.load_library_permanently("libsvml.dylib") elif sys.platform.startswith('win'): llvmbind.load_library_permanently("svml_dispmd") else: have_svml_library = False except: have_svml_library = False func = getattr(llvmbind.targets, "has_svml", None) llvm_svml_patched = func() if func is not None else False svml_operational = (config.USING_SVML and llvm_svml_patched \ and have_svml_library) print(fmt % ("SVML state, config.USING_SVML", config.USING_SVML)) print(fmt % ("SVML library found and loaded", have_svml_library)) print(fmt % ("llvmlite using SVML patched LLVM", llvm_svml_patched)) print(fmt % ("SVML operational", svml_operational)) # Check which threading backends are available. print("") print("__Threading Layer Information__") def parse_error(e, backend): # parses a linux based error message, this is to provide feedback # and hide user paths etc try: path, problem, symbol = [x.strip() for x in e.msg.split(':')] extn_dso = os.path.split(path)[1] if backend in extn_dso: return "%s: %s" % (problem, symbol) except Exception: pass return "Unknown import problem." try: from numba.npyufunc import tbbpool print(fmt % ("TBB Threading layer available", True)) except ImportError as e: # might be a missing symbol due to e.g. tbb libraries missing print(fmt % ("TBB Threading layer available", False)) print(fmt % ("+--> Disabled due to", parse_error(e, 'tbbpool'))) try: from numba.npyufunc import omppool print(fmt % ("OpenMP Threading layer available", True)) except ImportError as e: print(fmt % ("OpenMP Threading layer available", False)) print(fmt % ("+--> Disabled due to", parse_error(e, 'omppool'))) try: from numba.npyufunc import workqueue print(fmt % ("Workqueue Threading layer available", True)) except ImportError as e: print(fmt % ("Workqueue Threading layer available", False)) print(fmt % ("+--> Disabled due to", parse_error(e, 'workqueue'))) # look for numba env vars that are set print("") print("__Numba Environment Variable Information__") _envvar_found = False for k, v in os.environ.items(): if k.startswith('NUMBA_'): print(fmt % (k, v)) _envvar_found = True if not _envvar_found: print("None set.") # Look for conda and conda information print("") print("__Conda Information__") cmd = ["conda", "info", "--json"] try: conda_out = check_output(cmd) except Exception as e: print("Conda not present/not working.\nError was %s\n" % e) else: data = ''.join(conda_out.decode("utf-8").splitlines()) jsond = json.loads(data) keys = [ 'conda_build_version', 'conda_env_version', 'platform', 'python_version', 'root_writable' ] for k in keys: try: print(fmt % (k, jsond[k])) except KeyError: pass # get info about current environment cmd = ["conda", "list"] try: conda_out = check_output(cmd) except CalledProcessError as e: print("Error: Conda command failed. Error was %s\n" % e.output) else: print("") print("__Current Conda Env__") data = conda_out.decode("utf-8").splitlines() for k in data: if k[0] != '#': # don't show where the env is, personal data print(k) print("-" * 80) except Exception as e: print("Error: The system reporting tool has failed unexpectedly.") print("Exception was:") print(e) finally: print( "%s" % "If requested, please copy and paste the information between\n" "the dashed (----) lines, or from a given specific section as\n" "appropriate.\n\n" "=============================================================\n" "IMPORTANT: Please ensure that you are happy with sharing the\n" "contents of the information present, any information that you\n" "wish to keep private you should remove before sharing.\n" "=============================================================\n")
def get_sys_info(): # delay these imports until now as they are only needed in this # function which then exits. import platform import json import multiprocessing from numba import config from numba import cuda as cu from numba.cuda import cudadrv from numba.cuda.cudadrv.driver import driver as cudriver from numba import roc from numba.roc.hlc import hlc, libhlc import textwrap as tw import ctypes as ct import llvmlite.binding as llvmbind import locale from datetime import datetime from itertools import chain from subprocess import check_output, CalledProcessError try: fmt = "%-45s : %-s" print("-" * 80) print("__Time Stamp__") print(datetime.utcnow()) print("") print("__Hardware Information__") system_name = platform.system() print(fmt % ("Machine", platform.machine())) print(fmt % ("CPU Name", llvmbind.get_host_cpu_name())) if system_name == 'Linux': strmatch = 'Cpus_allowed' try: loc = '/proc/self/status' with open(loc, 'rt') as f: proc_stat = f.read().splitlines() for x in proc_stat: if x.startswith(strmatch): if x.startswith('%s:' % strmatch): hexnum = '0x%s' % x.split(':')[1].strip() acc_cpus = int(hexnum, 16) _n = str(bin(acc_cpus).count('1')) print(fmt % ("Number of accessible CPU cores", _n)) elif x.startswith('%s_list:' % strmatch): _a = x.split(':')[1].strip() print(fmt % ("Listed accessible CPUs cores", _a)) except BaseException: print(fmt % ("CPU count", multiprocessing.cpu_count())) # See if CFS is in place # https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt try: def scrape_lines(loc): with open(loc, 'rt') as f: return f.read().splitlines() loc = '/sys/fs/cgroup/cpuacct/cpu.cfs_period_us' cfs_period = int(scrape_lines(loc)[0]) loc = '/sys/fs/cgroup/cpuacct/cpu.cfs_quota_us' cfs_quota = int(scrape_lines(loc)[0]) if cfs_quota == -1: print(fmt % ("CFS restrictions", "None")) else: runtime_amount = float(cfs_quota)/float(cfs_period) print(fmt % ("CFS restrictions (CPUs worth of runtime)", runtime_amount)) except BaseException: print(fmt % ("CFS restrictions", 'Information not available')) else: print(fmt % ("CPU count", multiprocessing.cpu_count())) try: featuremap = llvmbind.get_host_cpu_features() except RuntimeError: print(fmt % ("CPU Features", "NA")) else: features = sorted([key for key, value in featuremap.items() if value]) cpu_feat = tw.fill(' '.join(features), 80) print(fmt % ("CPU Features", "")) print(cpu_feat) print("") print("__OS Information__") print(fmt % ("Platform", platform.platform(aliased=True))) print(fmt % ("Release", platform.release())) print(fmt % ("System Name", system_name)) print(fmt % ("Version", platform.version())) try: if system_name == 'Linux': info = platform.linux_distribution() elif system_name == 'Windows': info = platform.win32_ver() elif system_name == 'Darwin': info = platform.mac_ver() else: raise RuntimeError("Unknown system.") buf = ''.join([x if x != '' else ' ' for x in list(chain.from_iterable(info))]) print(fmt % ("OS specific info", buf)) if system_name == 'Linux': print(fmt % ("glibc info", ' '.join(platform.libc_ver()))) except: print("Error: System name incorrectly identified or unknown.") print("") print("__Python Information__") print(fmt % ("Python Compiler", platform.python_compiler())) print( fmt % ("Python Implementation", platform.python_implementation())) print(fmt % ("Python Version", platform.python_version())) lcl = [] try: for x in locale.getdefaultlocale(): if x is not None: lcl.append(x) except BaseException as e: lcl.append(str(e)) print(fmt % ("Python Locale ", ' '.join(lcl))) print("") print("__LLVM information__") print( fmt % ("LLVM version", '.'.join( [str(k) for k in llvmbind.llvm_version_info]))) print("") print("__CUDA Information__") # Look for GPUs try: cu.list_devices()[0] # will a device initialise? except BaseException as e: msg_not_found = "CUDA driver library cannot be found" msg_disabled_by_user = "******" msg_end = " or no CUDA enabled devices are present." msg_generic_problem = "Error: CUDA device intialisation problem." msg = getattr(e, 'msg', None) if msg is not None: if msg_not_found in msg: err_msg = msg_not_found + msg_end elif msg_disabled_by_user in msg: err_msg = msg_disabled_by_user + msg_end else: err_msg = msg_generic_problem + " Message:" + msg else: err_msg = msg_generic_problem + " " + str(e) # Best effort error report print("%s\nError class: %s" % (err_msg, str(type(e)))) else: try: cu.detect() dv = ct.c_int(0) cudriver.cuDriverGetVersion(ct.byref(dv)) print(fmt % ("CUDA driver version", dv.value)) print("CUDA libraries:") cudadrv.libs.test(sys.platform, print_paths=False) except: print( "Error: Probing CUDA failed (device and driver present, runtime problem?)\n") print("") print("__ROC Information__") roc_is_available = roc.is_available() print(fmt % ("ROC available", roc_is_available)) toolchains = [] try: libhlc.HLC() toolchains.append('librocmlite library') except: pass try: cmd = hlc.CmdLine().check_tooling() toolchains.append('ROC command line tools') except: pass # if no ROC try and report why if not roc_is_available: from numba.roc.hsadrv.driver import hsa try: hsa.is_available except BaseException as e: msg = str(e) else: msg = 'No ROC toolchains found.' print(fmt % ("Error initialising ROC due to", msg)) if toolchains: print(fmt % ("Available Toolchains", ', '.join(toolchains))) try: # ROC might not be available due to lack of tool chain, but HSA # agents may be listed from numba.roc.hsadrv.driver import hsa, dgpu_count decode = lambda x: x.decode('utf-8') if isinstance(x, bytes) else x print("\nFound %s HSA Agents:" % len(hsa.agents)) for i, agent in enumerate(hsa.agents): print('Agent id : %s' % i) print(' vendor: %s' % decode(agent.vendor_name)) print(' name: %s' % decode(agent.name)) print(' type: %s' % agent.device) print("") _dgpus = [] for a in hsa.agents: if a.is_component and a.device == 'GPU': _dgpus.append(decode(a.name)) print(fmt % ("Found %s discrete GPU(s)" % dgpu_count(), \ ', '.join(_dgpus))) except Exception as e: print("No HSA Agents found, encountered exception when searching:") print(e) print("") print("__SVML Information__") # replicate some SVML detection logic from numba.__init__ here. # if SVML load fails in numba.__init__ the splitting of the logic # here will help diagnosis of the underlying issue have_svml_library = True try: if sys.platform.startswith('linux'): llvmbind.load_library_permanently("libsvml.so") elif sys.platform.startswith('darwin'): llvmbind.load_library_permanently("libsvml.dylib") elif sys.platform.startswith('win'): llvmbind.load_library_permanently("svml_dispmd") else: have_svml_library = False except: have_svml_library = False func = getattr(llvmbind.targets, "has_svml", None) llvm_svml_patched = func() if func is not None else False svml_operational = (config.USING_SVML and llvm_svml_patched \ and have_svml_library) print(fmt % ("SVML state, config.USING_SVML", config.USING_SVML)) print(fmt % ("SVML library found and loaded", have_svml_library)) print(fmt % ("llvmlite using SVML patched LLVM", llvm_svml_patched)) print(fmt % ("SVML operational", svml_operational)) # Check which threading backends are available. print("") print("__Threading Layer Information__") def parse_error(e, backend): # parses a linux based error message, this is to provide feedback # and hide user paths etc try: path, problem, symbol = [x.strip() for x in e.msg.split(':')] extn_dso = os.path.split(path)[1] if backend in extn_dso: return "%s: %s" % (problem, symbol) except BaseException: pass return "Unknown import problem." try: from numba.npyufunc import tbbpool print(fmt % ("TBB Threading layer available", True)) except ImportError as e: # might be a missing symbol due to e.g. tbb libraries missing print(fmt % ("TBB Threading layer available", False)) print(fmt % ("+--> Disabled due to", parse_error(e, 'tbbpool'))) try: from numba.npyufunc import omppool print(fmt % ("OpenMP Threading layer available", True)) except ImportError as e: print(fmt % ("OpenMP Threading layer available", False)) print(fmt % ("+--> Disabled due to", parse_error(e, 'omppool'))) try: from numba.npyufunc import workqueue print(fmt % ("Workqueue Threading layer available", True)) except ImportError as e: print(fmt % ("Workqueue Threading layer available", False)) print(fmt % ("+--> Disabled due to", parse_error(e, 'workqueue'))) # look for numba env vars that are set print("") print("__Numba Environment Variable Information__") _envvar_found = False for k, v in os.environ.items(): if k.startswith('NUMBA_'): print(fmt % (k, v)) _envvar_found = True if not _envvar_found: print("None set.") # Look for conda and conda information print("") print("__Conda Information__") cmd = ["conda", "info", "--json"] try: conda_out = check_output(cmd) except Exception as e: print( "Conda not present/not working.\nError was %s\n" % e) else: data = ''.join(conda_out.decode("utf-8").splitlines()) jsond = json.loads(data) keys = ['conda_build_version', 'conda_env_version', 'platform', 'python_version', 'root_writable'] for k in keys: try: print(fmt % (k, jsond[k])) except KeyError: pass # get info about current environment cmd = ["conda", "list"] try: conda_out = check_output(cmd) except CalledProcessError as e: print("Error: Conda command failed. Error was %s\n" % e.output) else: print("") print("__Current Conda Env__") data = conda_out.decode("utf-8").splitlines() for k in data: if k[0] != '#': # don't show where the env is, personal data print(k) print("-" * 80) except Exception as e: print("Error: The system reporting tool has failed unexpectedly.") print("Exception was:") print(e) finally: print( "%s" % "If requested, please copy and paste the information between\n" "the dashed (----) lines, or from a given specific section as\n" "appropriate.\n\n" "=============================================================\n" "IMPORTANT: Please ensure that you are happy with sharing the\n" "contents of the information present, any information that you\n" "wish to keep private you should remove before sharing.\n" "=============================================================\n")
def get_sys_info(): # delay these imports until now as they are only needed in this # function which then exits. import platform import json from numba import config from numba import cuda as cu from numba.cuda import cudadrv from numba.cuda.cudadrv.driver import driver as cudriver from numba import roc from numba.roc.hlc import hlc, libhlc import textwrap as tw import ctypes as ct import llvmlite.binding as llvmbind import locale from datetime import datetime from itertools import chain from subprocess import check_output, CalledProcessError try: fmt = "%-35s : %-s" print("-" * 80) print("__Time Stamp__") print(datetime.utcnow()) print("") print("__Hardware Information__") print(fmt % ("Machine", platform.machine())) print(fmt % ("CPU Name", llvmbind.get_host_cpu_name())) try: featuremap = llvmbind.get_host_cpu_features() except RuntimeError: print(fmt % ("CPU Features", "NA")) else: features = sorted( [key for key, value in featuremap.items() if value]) cpu_feat = tw.fill(' '.join(features), 80) print(fmt % ("CPU Features", "")) print(cpu_feat) print("") print("__OS Information__") print(fmt % ("Platform", platform.platform(aliased=True))) print(fmt % ("Release", platform.release())) system_name = platform.system() print(fmt % ("System Name", system_name)) print(fmt % ("Version", platform.version())) try: if system_name == 'Linux': info = platform.linux_distribution() elif system_name == 'Windows': info = platform.win32_ver() elif system_name == 'Darwin': info = platform.mac_ver() else: raise RuntimeError("Unknown system.") buf = ''.join([ x if x != '' else ' ' for x in list(chain.from_iterable(info)) ]) print(fmt % ("OS specific info", buf)) if system_name == 'Linux': print(fmt % ("glibc info", ' '.join(platform.libc_ver()))) except: print("Error: System name incorrectly identified or unknown.") print("") print("__Python Information__") print(fmt % ("Python Compiler", platform.python_compiler())) print(fmt % ("Python Implementation", platform.python_implementation())) print(fmt % ("Python Version", platform.python_version())) print(fmt % ("Python Locale ", ' '.join( [x for x in locale.getdefaultlocale() if x is not None]))) print("") print("__LLVM information__") print(fmt % ("LLVM version", '.'.join( [str(k) for k in llvmbind.llvm_version_info]))) print("") print("__CUDA Information__") # Look for GPUs try: cu.list_devices()[0] # will a device initialise? except BaseException as e: msg_not_found = "CUDA driver library cannot be found" msg_disabled_by_user = "******" msg_end = " or no CUDA enabled devices are present." msg_generic_problem = "Error: CUDA device intialisation problem." msg = getattr(e, 'msg', None) if msg is not None: if msg_not_found in msg: err_msg = msg_not_found + msg_end elif msg_disabled_by_user in msg: err_msg = msg_disabled_by_user + msg_end else: err_msg = msg_generic_problem + " Message:" + msg else: err_msg = msg_generic_problem + " " + str(e) # Best effort error report print("%s\nError class: %s" % (err_msg, str(type(e)))) else: try: cu.detect() dv = ct.c_int(0) cudriver.cuDriverGetVersion(ct.byref(dv)) print(fmt % ("CUDA driver version", dv.value)) print("CUDA libraries:") cudadrv.libs.test(sys.platform, print_paths=False) except: print( "Error: Probing CUDA failed (device and driver present, runtime problem?)\n" ) print("") print("__ROC Information__") roc_is_available = roc.is_available() print(fmt % ("ROC available", roc_is_available)) toolchains = [] try: libhlc.HLC() toolchains.append('librocmlite library') except: pass try: cmd = hlc.CmdLine().check_tooling() toolchains.append('ROC command line tools') except: pass # if no ROC try and report why if not roc_is_available: from numba.roc.hsadrv.driver import hsa try: hsa.is_available except BaseException as e: msg = str(e) else: msg = 'No ROC toolchains found.' print(fmt % ("Error initialising ROC due to", msg)) if toolchains: print(fmt % ("Available Toolchains", ', '.join(toolchains))) try: # ROC might not be available due to lack of tool chain, but HSA # agents may be listed from numba.roc.hsadrv.driver import hsa, dgpu_count print("\nFound %s HSA Agents:" % len(hsa.agents)) for i, agent in enumerate(hsa.agents): print('Agent id : %s' % i) print(' vendor: %s' % agent.vendor_name) print(' name: %s' % agent.name) print(' type: %s' % agent.device) print("") _dgpus = [] for a in hsa.agents: if a.is_component and a.device == 'GPU': _dgpus.append(a.name) print(fmt % ("Found %s discrete GPU(s)" % dgpu_count(), \ ', '.join(_dgpus))) except Exception as e: print("No HSA Agents found, encountered exception when searching:") print(e) print("") print("__SVML Information__") # replicate some SVML detection logic from numba.__init__ here. # if SVML load fails in numba.__init__ the splitting of the logic # here will help diagnosis of the underlying issue have_svml_library = True try: if sys.platform.startswith('linux'): llvmbind.load_library_permanently("libsvml.so") elif sys.platform.startswith('darwin'): llvmbind.load_library_permanently("libsvml.dylib") elif sys.platform.startswith('win'): llvmbind.load_library_permanently("svml_dispmd") else: have_svml_library = False except: have_svml_library = False func = getattr(llvmbind.targets, "has_svml", None) llvm_svml_patched = func() if func is not None else False svml_operational = (config.USING_SVML and llvm_svml_patched \ and have_svml_library) print(fmt % ("SVML state, config.USING_SVML", config.USING_SVML)) print(fmt % ("SVML library found and loaded", have_svml_library)) print(fmt % ("llvmlite using SVML patched LLVM", llvm_svml_patched)) print(fmt % ("SVML operational:", svml_operational)) # Look for conda and conda information print("") print("__Conda Information__") cmd = ["conda", "info", "--json"] try: conda_out = check_output(cmd) except Exception as e: print("Conda not present/not working.\nError was %s\n" % e) else: data = ''.join(conda_out.decode("utf-8").splitlines()) jsond = json.loads(data) keys = [ 'conda_build_version', 'conda_env_version', 'platform', 'python_version', 'root_writable' ] for k in keys: try: print(fmt % (k, jsond[k])) except KeyError: pass # get info about current environment cmd = ["conda", "list"] try: conda_out = check_output(cmd) except CalledProcessError as e: print("Error: Conda command failed. Error was %s\n" % e.output) else: print("") print("__Current Conda Env__") data = conda_out.decode("utf-8").splitlines() for k in data: if k[0] != '#': # don't show where the env is, personal data print(k) print("-" * 80) except Exception as e: print("Error: The system reporting tool has failed unexpectedly.") print("Exception was:") print(e) finally: print( "%s" % "If requested, please copy and paste the information between\n" "the dashed (----) lines, or from a given specific section as\n" "appropriate.\n\n" "=============================================================\n" "IMPORTANT: Please ensure that you are happy with sharing the\n" "contents of the information present, any information that you\n" "wish to keep private you should remove before sharing.\n" "=============================================================\n")
def get_target_machine(self): """Instantiate and return target machine.""" features = llvm.get_host_cpu_features().flatten() cpu = '' # llvm.get_host_cpu_name() # Work around until ryzen problems are fixed return llvm.Target.from_default_triple().create_target_machine( cpu=cpu, features=features, opt=3)