def main():
libartiq_support = os.getenv("LIBARTIQ_SUPPORT")
if libartiq_support is not None:
llvm.load_library_permanently(libartiq_support)
def process_diagnostic(diag):
print("\n".join(diag.render()))
if diag.level in ("fatal", "error"):
exit(1)
engine = diagnostic.Engine()
engine.process = process_diagnostic
source = "".join(fileinput.input())
source = source.replace("#ARTIQ#", "")
mod = Module(Source.from_string(source.expandtabs(), engine=engine))
target = NativeTarget()
llmod = mod.build_llvm_ir(target)
llparsedmod = llvm.parse_assembly(str(llmod))
llparsedmod.verify()
llmachine = llvm.Target.from_triple(target.triple).create_target_machine()
lljit = llvm.create_mcjit_compiler(llparsedmod, llmachine)
llmain = lljit.get_function_address(llmod.name + ".__modinit__")
ctypes.CFUNCTYPE(None)(llmain)()
def test_libm(self):
system = platform.system()
if system == "Linux":
libm = find_library("m")
elif system == "Darwin":
libm = find_library("libm")
llvm.load_library_permanently(libm)
def printf(builder, fmt, *args, override_debug=False):
if "print_values" not in debug_env and not override_debug:
return
#FIXME: Fix builtin printf and use that instead of this
try:
import llvmlite.binding as llvm
libc = util.find_library("c")
llvm.load_library_permanently(libc)
# Address will be none if the symbol is not found
printf_address = llvm.address_of_symbol("printf")
except Exception as e:
return
# Direct pointer constants don't work
printf_ty = ir.FunctionType(ir.IntType(32), [ir.IntType(8).as_pointer()],
var_arg=True)
printf = builder.inttoptr(
ir.IntType(64)(printf_address), printf_ty.as_pointer())
ir_module = builder.function.module
fmt += "\0"
int8 = ir.IntType(8)
fmt_data = bytearray(fmt.encode("utf8"))
fmt_ty = ir.ArrayType(int8, len(fmt_data))
global_fmt = ir.GlobalVariable(ir_module,
fmt_ty,
name="printf_fmt_" +
str(len(ir_module.globals)))
global_fmt.linkage = "internal"
global_fmt.global_constant = True
global_fmt.initializer = fmt_ty(fmt_data)
fmt_ptr = builder.gep(global_fmt, [ir.IntType(32)(0), ir.IntType(32)(0)])
builder.call(printf, [fmt_ptr] + list(args))
def run_llvm(ops):
llvm.load_library_permanently("./libio.so")
mem_size = 10 ** 6
import numpy as np
arr = np.zeros(mem_size, dtype=np.int8)
mem = arr.ctypes.data_as(POINTER(c_int8))
fnty = ir.FunctionType(int8, (int8.as_pointer(),))
module = ir.Module(name="module_name")
func = ir.Function(module, fnty, name="foo")
write_fnty = ir.FunctionType(void, (int8, int32))
write_func = ir.Function(module, write_fnty, name="sys_write")
read_fnty = ir.FunctionType(int8, (int32,))
read_func = ir.Function(module, read_fnty, name="sys_read")
ptr = ir.GlobalVariable(module, int8_ptr, "ptr")
ptr.linkage = 'internal'
to_llvm(ops, func, ptr, write_func, read_func)
target_machine, engine = create_execution_engine()
print(module)
mod = compile_ir(engine, str(module))
print(target_machine.emit_assembly(mod))
func_ptr = engine.get_function_address("foo")
cfunc = CFUNCTYPE(c_int8, POINTER(c_int8))(func_ptr)
cfunc(mem)
def _generate_cpu_printf_wrapper(module):
printf_ty = ir.FunctionType(ir.IntType(32), [ir.IntType(8).as_pointer()],
var_arg=True)
function = ir.Function(module, printf_ty, name=_BUILTIN_PREFIX + "printf")
function.attributes.add('alwaysinline')
block = function.append_basic_block(name="entry")
builder = ir.IRBuilder(block)
builder.debug_metadata = LLVMBuilderContext.get_debug_location(
function, None)
try:
import llvmlite.binding as llvm
libc = ctypes.util.find_library("c")
llvm.load_library_permanently(libc)
# Address will be none if the symbol is not found
printf_address = llvm.address_of_symbol("printf")
except:
printf_address = None
if printf_address is not None:
# Direct pointer constants don't work
printf = builder.inttoptr(
pnlvm.ir.IntType(64)(printf_address), printf_ty.as_pointer())
builder.ret(builder.call(printf, function.args))
else:
builder.ret(ir.IntType(32)(-1))
def init(self):
self.is32bit = (utils.MACHINE_BITS == 32)
self._internal_codegen = codegen.JITCPUCodegen("numba.exec")
# Add ARM ABI functions from libgcc_s
if platform.machine() == 'armv7l':
ll.load_library_permanently('libgcc_s.so.1')
# Map external C functions.
externals.c_math_functions.install(self)
# Initialize NRT runtime
rtsys.initialize(self)
# Initialize additional implementations
import numba.cpython.unicode
import numba.cpython.charseq
import numba.typed.dictimpl
import numba.experimental.function_type
# Add lower_extension attribute
self.lower_extensions = {}
from numba.parfors.parfor_lowering import _lower_parfor_parallel
from numba.parfors.parfor import Parfor
# Specify how to lower Parfor nodes using the lower_extensions
self.lower_extensions[Parfor] = _lower_parfor_parallel
def load_dpctl_sycl_interface():
"""Permanently loads the ``DPCTLSyclInterface`` library provided by dpctl.
The ``DPCTLSyclInterface`` library provides C wrappers over SYCL functions
that are directly invoked from the LLVM modules generated by numba-dppy.
We load the library once at the time of initialization using llvmlite's
load_library_permanently function.
Raises:
ImportError: If the ``DPCTLSyclInterface`` library could not be loaded.
"""
import glob
import platform as plt
import dpctl
platform = plt.system()
if platform == "Windows":
paths = glob.glob(
os.path.join(os.path.dirname(dpctl.__file__),
"*DPCTLSyclInterface.dll"))
else:
paths = glob.glob(
os.path.join(os.path.dirname(dpctl.__file__),
"*DPCTLSyclInterface.so"))
if len(paths) == 1:
ll.load_library_permanently(paths[0])
else:
raise ImportError
def init_dppy_vectorize():
return DPPYVectorize
Vectorize.target_registry.ondemand["dppy"] = init_dppy_vectorize
def run_tests():
for name in test_inputs:
parser = SLRParser("new_grammar.txt", '../testInputs/' + name + '.txt')
syn_tree = parser.parse()[0]
syn_tree.genCases(parser.grammar)
sym_table = SymbolTable()
named_sym_table = NamedSymbolTable()
valid = errorCheck(syn_tree, sym_table, named_sym_table,
parser.lexer.infileLines)
llvm_custom_types = create_llvm_types(named_sym_table)
module = ir.Module(name=name)
generated_ir = codegen(syn_tree, named_sym_table, {
'mod': module,
'llvm_custom_types': llvm_custom_types
})
binding.initialize()
binding.initialize_native_target()
binding.initialize_native_asmprinter() # yes, even this one
binding.load_library_permanently('../runtime.so')
binding.parse_assembly(str(module), None)
engine = create_execution_engine()
mod = compile_ir(engine, str(module))
test_inputs[name](engine)
def Codegen(ast: ProgramNode, printIR: bool = True):
"""Genera l'LLVM IR dall'AST in input.
Args:
ast: L'albero di sintassi astratto.
printIR (bool): Un boolean che indica se fare un print dell'IR generato (default `False`)
Returns:
L'IR generato sotto forma di `str`
Raises:
StdlibNotFoundError: Se la standard lib non puo' essere trovata
"""
llvm.initialize()
llvm.initialize_native_target()
llvm.initialize_native_asmprinter()
# Decide l'estensione della stdlib
ext = None
if platform.system() == 'Windows':
ext = 'dll'
elif platform.system() == 'Darwin':
ext = 'dylib'
elif platform.system() == 'Linux':
ext = 'so'
path = pathlib.Path(__file__).parent.absolute()
try:
llvm.load_library_permanently(
str(path) + "/../../bin/tiny_hi_core.{}".format(ext))
except Exception:
print("StdlibNotFoundError: Cannot find the standard library (tiny_hi_core.{})".format(ext))
return (None, None)
module = Module()
builder = IRBuilder()
context = Context(module)
entry = None
llmod = None
try:
entry = ast.entry_point()
ast.codegen(builder, context)
strmod = str(module)
if printIR:
print(strmod)
llmod = llvm.parse_assembly(strmod)
except Exception as e:
print("Codegen Failed")
print("{}: {}".format(type(e).__name__, e))
finally:
return (llmod, entry)
def execution_engine(self):
target = binding.Target.from_default_triple()
target_machine = target.create_target_machine()
backing_mod = binding.parse_assembly("")
engine = binding.create_mcjit_compiler(backing_mod, target_machine)
return engine
binding.load_library_permanently("//funcoes.so")
def bindAndRun(self):
print("Binding to LLVM and running. Here is the LLVM to be run:")
llvm.load_library_permanently(r"./runtime/runtimelib.so")
void = self.getType("void")
# self.builder.ret_void()
self.builder.ret(self.getType("true"))
#print(self.builder.basic_block)
llvm.initialize()
llvm.initialize_native_target()
llvm.initialize_native_asmprinter() # yes, even this one
llvm_ir = self.getIR()
print("------------------Output-----------------------")
def create_execution_engine():
"""
Create an ExecutionEngine suitable for JIT code generation on
the host CPU. The engine is reusable for an arbitrary number of
modules.
"""
# Create a target machine representing the host
target = llvm.Target.from_default_triple()
target_machine = target.create_target_machine()
# And an execution engine with an empty backing module
backing_mod = llvm.parse_assembly("")
engine = llvm.create_mcjit_compiler(backing_mod, target_machine)
return engine
def compile_ir(engine, llvm_ir):
"""
Compile the LLVM IR string with the given engine.
The compiled module object is returned.
"""
# Create a LLVM module object from the IR
mod = llvm.parse_assembly(llvm_ir)
mod.verify()
# Now add the module and make sure it is ready for execution
engine.add_module(mod)
engine.finalize_object()
engine.run_static_constructors()
return mod
engine = create_execution_engine()
mod = compile_ir(engine, llvm_ir)
# Look up the function pointer (a Python int)
func_ptr = engine.get_function_address("main")
# Run the function via ctypes
cfunc = CFUNCTYPE(c_bool)(func_ptr)
res = cfunc()
print("Exit: ", res)
def open_daal4py():
'''open daal4py library and load C-symbols'''
import os
import glob
path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(daal4py.__file__))), '_daal4py.c*')
lib = glob.glob(path)
assert len(lib) == 1
# just load the whole thing
ll.load_library_permanently(lib[0])
def load_lfortran_runtime_library():
"""
Locates and loads the LFortran runtime library.
If the library is already loaded, it will not load it again.
"""
global _lfortran_runtime_library_loaded
if not _lfortran_runtime_library_loaded:
base_dir = get_lib_path()
liblfortran_so = os.path.join(base_dir, "liblfortran.so")
llvm.load_library_permanently(liblfortran_so)
_lfortran_runtime_library_loaded = True
def make_execution_engine():
"""
Initialize just-in-time execution engine.
:rtype: ExecutionEngine
"""
target = llvm.Target.from_default_triple()
target_machine = target.create_target_machine()
backing_mod = llvm.parse_assembly('')
engine = llvm.create_mcjit_compiler(backing_mod, target_machine)
llvm.load_library_permanently(PREAMBLE_BINARIES_PATH + '.so')
yield engine
del engine
def init_jit(libc_path):
global jit
"""
Create an ExecutionEngine suitable for JIT code generation on
the host CPU. The engine is reusable for an arbitrary number of
modules.
"""
# Create a target machine representing the host
target_machine = globals.target_machine
# And an execution engine with an empty backing module
backing_mod = llvm.parse_assembly("")
jit = llvm.create_mcjit_compiler(backing_mod, globals.target_machine)
# Add libc
llvm.load_library_permanently(libc_path)
def init(self):
self.is32bit = (utils.MACHINE_BITS == 32)
self._internal_codegen = codegen.JITCPUCodegen("numba.exec")
# Add ARM ABI functions from libgcc_s
if platform.machine() == 'armv7l':
ll.load_library_permanently('libgcc_s.so.1')
# Map external C functions.
externals.c_math_functions.install(self)
# Initialize NRT runtime
rtsys.initialize(self)
# Initialize additional implementations
if utils.PY3:
import numba.unicode
def init(self):
self.is32bit = (utils.MACHINE_BITS == 32)
self._internal_codegen = codegen.JITCPUCodegen("numba.exec")
# Add ARM ABI functions from libgcc_s
if platform.machine() == 'armv7l':
ll.load_library_permanently('libgcc_s.so.1')
# Map external C functions.
externals.c_math_functions.install(self)
# Initialize NRT runtime
rtsys.initialize(self)
# Initialize additional implementations
if utils.PY3:
import numba.unicode
def init(self):
self.is32bit = utils.MACHINE_BITS == 32
self._internal_codegen = codegen.JITCPUCodegen("numba.exec")
# Add ARM ABI functions from libgcc_s
if platform.machine() == "armv7l":
ll.load_library_permanently("libgcc_s.so.1")
# Map external C functions.
externals.c_math_functions.install(self)
# Initialize NRT runtime
rtsys.initialize(self)
# Initialize additional implementations
import numba.cpython.unicode
import numba.typed.dictimpl
import numba.experimental.function_type
def load_lfortran_runtime_library():
"""
Locates and loads the LFortran runtime library.
If the library is already loaded, it will not load it again.
"""
global _lfortran_runtime_library_loaded
if not _lfortran_runtime_library_loaded:
base_dir = get_lib_path()
if sys.platform == "linux":
shprefix = "lib"
shsuffix = "so"
elif sys.platform == "win32":
shprefix = ""
shsuffix = "dll"
elif sys.platform == "darwin":
shprefix = "lib"
shsuffix = "dylib"
else:
raise NotImplementedError("Platform not supported yet.")
liblfortran_so = os.path.join(base_dir, shprefix + "lfortran_runtime." \
+ shsuffix)
llvm.load_library_permanently(liblfortran_so)
_lfortran_runtime_library_loaded = True
import re
import subprocess
import sys
import unittest
from contextlib import contextmanager
from tempfile import mkstemp
from llvmlite import six, ir
from llvmlite import binding as llvm
from llvmlite.binding import ffi
from . import TestCase
# arvm7l needs extra ABI symbols to link successfully
if platform.machine() == 'armv7l':
llvm.load_library_permanently('libgcc_s.so.1')
def no_de_locale():
cur = locale.setlocale(locale.LC_ALL)
try:
locale.setlocale(locale.LC_ALL, 'de_DE')
except locale.Error:
return True
else:
return False
finally:
locale.setlocale(locale.LC_ALL, cur)
asm_sum = r"""
from llvmlite import binding as llvm
from semantica import *
llvm.initialize()
llvm.initialize_all_targets()
llvm.initialize_native_target()
llvm.initialize_native_asmprinter()
module = ir.Module('meu_modulo.bc')
module.triple = llvm.get_default_triple()
target = llvm.Target.from_triple(module.triple)
target_machine = target.create_target_machine()
llvm.load_library_permanently('./io.so')
module.data_layout = target_machine.target_data
escrevaInteiro = ir.Function(module,ir.FunctionType(ir.VoidType(), [ir.IntType(32)]),name="escrevaInteiro")
escrevaFlutuante = ir.Function(module,ir.FunctionType(ir.VoidType(),[ir.FloatType()]),name="escrevaFlutuante")
leiaInteiro = ir.Function(module,ir.FunctionType(ir.IntType(32),[]),name="leiaInteiro")
leiaFlutuante = ir.Function(module,ir.FunctionType(ir.FloatType(),[]),name="leiaFlutuante")
vetTabela = []
# funcNow = {
# "nome": None,
# "escopo": "global",
# "type": None,
# "funcCabecalho": None,
# "entryBlock": None,
def get_sys_info():
# delay these imports until now as they are only needed in this
# function which then exits.
import platform
import json
from numba import config
from numba import cuda as cu
from numba.cuda import cudadrv
from numba.cuda.cudadrv.driver import driver as cudriver
from numba import roc
from numba.roc.hlc import hlc, libhlc
import textwrap as tw
import ctypes as ct
import llvmlite.binding as llvmbind
import locale
from datetime import datetime
from itertools import chain
from subprocess import check_output, CalledProcessError
try:
fmt = "%-35s : %-s"
print("-" * 80)
print("__Time Stamp__")
print(datetime.utcnow())
print("")
print("__Hardware Information__")
print(fmt % ("Machine", platform.machine()))
print(fmt % ("CPU Name", llvmbind.get_host_cpu_name()))
try:
featuremap = llvmbind.get_host_cpu_features()
except RuntimeError:
print(fmt % ("CPU Features", "NA"))
else:
features = sorted([key for key, value in featuremap.items()
if value])
cpu_feat = tw.fill(' '.join(features), 80)
print(fmt % ("CPU Features", ""))
print(cpu_feat)
print("")
print("__OS Information__")
print(fmt % ("Platform", platform.platform(aliased=True)))
print(fmt % ("Release", platform.release()))
system_name = platform.system()
print(fmt % ("System Name", system_name))
print(fmt % ("Version", platform.version()))
try:
if system_name == 'Linux':
info = platform.linux_distribution()
elif system_name == 'Windows':
info = platform.win32_ver()
elif system_name == 'Darwin':
info = platform.mac_ver()
else:
raise RuntimeError("Unknown system.")
buf = ''.join([x
if x != '' else ' '
for x in list(chain.from_iterable(info))])
print(fmt % ("OS specific info", buf))
if system_name == 'Linux':
print(fmt % ("glibc info", ' '.join(platform.libc_ver())))
except:
print("Error: System name incorrectly identified or unknown.")
print("")
print("__Python Information__")
print(fmt % ("Python Compiler", platform.python_compiler()))
print(
fmt %
("Python Implementation",
platform.python_implementation()))
print(fmt % ("Python Version", platform.python_version()))
print(
fmt %
("Python Locale ", ' '.join(
[x for x in locale.getdefaultlocale() if x is not None])))
print("")
print("__LLVM information__")
print(
fmt %
("LLVM version", '.'.join(
[str(k) for k in llvmbind.llvm_version_info])))
print("")
print("__CUDA Information__")
# Look for GPUs
try:
cu.list_devices()[0] # will a device initialise?
except BaseException as e:
msg_not_found = "CUDA driver library cannot be found"
msg_disabled_by_user = "******"
msg_end = " or no CUDA enabled devices are present."
msg_generic_problem = "Error: CUDA device intialisation problem."
msg = getattr(e, 'msg', None)
if msg is not None:
if msg_not_found in msg:
err_msg = msg_not_found + msg_end
elif msg_disabled_by_user in msg:
err_msg = msg_disabled_by_user + msg_end
else:
err_msg = msg_generic_problem + " Message:" + msg
else:
err_msg = msg_generic_problem + " " + str(e)
# Best effort error report
print("%s\nError class: %s" % (err_msg, str(type(e))))
else:
try:
cu.detect()
dv = ct.c_int(0)
cudriver.cuDriverGetVersion(ct.byref(dv))
print(fmt % ("CUDA driver version", dv.value))
print("CUDA libraries:")
cudadrv.libs.test(sys.platform, print_paths=False)
except:
print(
"Error: Probing CUDA failed (device and driver present, runtime problem?)\n")
print("")
print("__ROC Information__")
roc_is_available = roc.is_available()
print(fmt % ("ROC available", roc_is_available))
toolchains = []
try:
libhlc.HLC()
toolchains.append('librocmlite library')
except:
pass
try:
cmd = hlc.CmdLine().check_tooling()
toolchains.append('ROC command line tools')
except:
pass
# if no ROC try and report why
if not roc_is_available:
from numba.roc.hsadrv.driver import hsa
try:
hsa.is_available
except BaseException as e:
msg = str(e)
else:
msg = 'No ROC toolchains found.'
print(fmt % ("Error initialising ROC due to", msg))
if toolchains:
print(fmt % ("Available Toolchains", ', '.join(toolchains)))
try:
# ROC might not be available due to lack of tool chain, but HSA
# agents may be listed
from numba.roc.hsadrv.driver import hsa, dgpu_count
decode = lambda x: x.decode('utf-8') if isinstance(x, bytes) else x
print("\nFound %s HSA Agents:" % len(hsa.agents))
for i, agent in enumerate(hsa.agents):
print('Agent id : %s' % i)
print(' vendor: %s' % decode(agent.vendor_name))
print(' name: %s' % decode(agent.name))
print(' type: %s' % agent.device)
print("")
_dgpus = []
for a in hsa.agents:
if a.is_component and a.device == 'GPU':
_dgpus.append(decode(a.name))
print(fmt % ("Found %s discrete GPU(s)" % dgpu_count(), \
', '.join(_dgpus)))
except Exception as e:
print("No HSA Agents found, encountered exception when searching:")
print(e)
print("")
print("__SVML Information__")
# replicate some SVML detection logic from numba.__init__ here.
# if SVML load fails in numba.__init__ the splitting of the logic
# here will help diagnosis of the underlying issue
have_svml_library = True
try:
if sys.platform.startswith('linux'):
llvmbind.load_library_permanently("libsvml.so")
elif sys.platform.startswith('darwin'):
llvmbind.load_library_permanently("libsvml.dylib")
elif sys.platform.startswith('win'):
llvmbind.load_library_permanently("svml_dispmd")
else:
have_svml_library = False
except:
have_svml_library = False
func = getattr(llvmbind.targets, "has_svml", None)
llvm_svml_patched = func() if func is not None else False
svml_operational = (config.USING_SVML and llvm_svml_patched \
and have_svml_library)
print(fmt % ("SVML state, config.USING_SVML", config.USING_SVML))
print(fmt % ("SVML library found and loaded", have_svml_library))
print(fmt % ("llvmlite using SVML patched LLVM", llvm_svml_patched))
print(fmt % ("SVML operational:", svml_operational))
# Look for conda and conda information
print("")
print("__Conda Information__")
cmd = ["conda", "info", "--json"]
try:
conda_out = check_output(cmd)
except Exception as e:
print(
"Conda not present/not working.\nError was %s\n" % e)
else:
data = ''.join(conda_out.decode("utf-8").splitlines())
jsond = json.loads(data)
keys = ['conda_build_version',
'conda_env_version',
'platform',
'python_version',
'root_writable']
for k in keys:
try:
print(fmt % (k, jsond[k]))
except KeyError:
pass
# get info about current environment
cmd = ["conda", "list"]
try:
conda_out = check_output(cmd)
except CalledProcessError as e:
print("Error: Conda command failed. Error was %s\n" % e.output)
else:
print("")
print("__Current Conda Env__")
data = conda_out.decode("utf-8").splitlines()
for k in data:
if k[0] != '#': # don't show where the env is, personal data
print(k)
print("-" * 80)
except Exception as e:
print("Error: The system reporting tool has failed unexpectedly.")
print("Exception was:")
print(e)
finally:
print(
"%s" %
"If requested, please copy and paste the information between\n"
"the dashed (----) lines, or from a given specific section as\n"
"appropriate.\n\n"
"=============================================================\n"
"IMPORTANT: Please ensure that you are happy with sharing the\n"
"contents of the information present, any information that you\n"
"wish to keep private you should remove before sharing.\n"
"=============================================================\n")
def get_sys_info():
# delay these imports until now as they are only needed in this
# function which then exits.
import platform
import json
import multiprocessing
from numba import config
from numba import cuda as cu
from numba.cuda import cudadrv
from numba.cuda.cudadrv.driver import driver as cudriver
from numba import roc
from numba.roc.hlc import hlc, libhlc
import textwrap as tw
import ctypes as ct
import llvmlite.binding as llvmbind
import locale
from datetime import datetime
from itertools import chain
from subprocess import check_output, CalledProcessError
try:
fmt = "%-45s : %-s"
print("-" * 80)
print("__Time Stamp__")
print(datetime.utcnow())
print("")
print("__Hardware Information__")
system_name = platform.system()
print(fmt % ("Machine", platform.machine()))
print(fmt % ("CPU Name", llvmbind.get_host_cpu_name()))
if system_name == 'Linux':
strmatch = 'Cpus_allowed'
try:
loc = '/proc/self/status'
with open(loc, 'rt') as f:
proc_stat = f.read().splitlines()
for x in proc_stat:
if x.startswith(strmatch):
if x.startswith('%s:' % strmatch):
hexnum = '0x%s' % x.split(':')[1].strip()
acc_cpus = int(hexnum, 16)
_n = str(bin(acc_cpus).count('1'))
print(fmt % ("Number of accessible CPU cores",
_n))
elif x.startswith('%s_list:' % strmatch):
_a = x.split(':')[1].strip()
print(fmt % ("Listed accessible CPUs cores",
_a))
except BaseException:
print(fmt % ("CPU count", multiprocessing.cpu_count()))
# See if CFS is in place
# https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
try:
def scrape_lines(loc):
with open(loc, 'rt') as f:
return f.read().splitlines()
loc = '/sys/fs/cgroup/cpuacct/cpu.cfs_period_us'
cfs_period = int(scrape_lines(loc)[0])
loc = '/sys/fs/cgroup/cpuacct/cpu.cfs_quota_us'
cfs_quota = int(scrape_lines(loc)[0])
if cfs_quota == -1:
print(fmt % ("CFS restrictions", "None"))
else:
runtime_amount = float(cfs_quota)/float(cfs_period)
print(fmt % ("CFS restrictions (CPUs worth of runtime)",
runtime_amount))
except BaseException:
print(fmt % ("CFS restrictions", 'Information not available'))
else:
print(fmt % ("CPU count", multiprocessing.cpu_count()))
try:
featuremap = llvmbind.get_host_cpu_features()
except RuntimeError:
print(fmt % ("CPU Features", "NA"))
else:
features = sorted([key for key, value in featuremap.items()
if value])
cpu_feat = tw.fill(' '.join(features), 80)
print(fmt % ("CPU Features", ""))
print(cpu_feat)
print("")
print("__OS Information__")
print(fmt % ("Platform", platform.platform(aliased=True)))
print(fmt % ("Release", platform.release()))
print(fmt % ("System Name", system_name))
print(fmt % ("Version", platform.version()))
try:
if system_name == 'Linux':
info = platform.linux_distribution()
elif system_name == 'Windows':
info = platform.win32_ver()
elif system_name == 'Darwin':
info = platform.mac_ver()
else:
raise RuntimeError("Unknown system.")
buf = ''.join([x
if x != '' else ' '
for x in list(chain.from_iterable(info))])
print(fmt % ("OS specific info", buf))
if system_name == 'Linux':
print(fmt % ("glibc info", ' '.join(platform.libc_ver())))
except:
print("Error: System name incorrectly identified or unknown.")
print("")
print("__Python Information__")
print(fmt % ("Python Compiler", platform.python_compiler()))
print(
fmt %
("Python Implementation",
platform.python_implementation()))
print(fmt % ("Python Version", platform.python_version()))
lcl = []
try:
for x in locale.getdefaultlocale():
if x is not None:
lcl.append(x)
except BaseException as e:
lcl.append(str(e))
print(fmt % ("Python Locale ", ' '.join(lcl)))
print("")
print("__LLVM information__")
print(
fmt %
("LLVM version", '.'.join(
[str(k) for k in llvmbind.llvm_version_info])))
print("")
print("__CUDA Information__")
# Look for GPUs
try:
cu.list_devices()[0] # will a device initialise?
except BaseException as e:
msg_not_found = "CUDA driver library cannot be found"
msg_disabled_by_user = "******"
msg_end = " or no CUDA enabled devices are present."
msg_generic_problem = "Error: CUDA device intialisation problem."
msg = getattr(e, 'msg', None)
if msg is not None:
if msg_not_found in msg:
err_msg = msg_not_found + msg_end
elif msg_disabled_by_user in msg:
err_msg = msg_disabled_by_user + msg_end
else:
err_msg = msg_generic_problem + " Message:" + msg
else:
err_msg = msg_generic_problem + " " + str(e)
# Best effort error report
print("%s\nError class: %s" % (err_msg, str(type(e))))
else:
try:
cu.detect()
dv = ct.c_int(0)
cudriver.cuDriverGetVersion(ct.byref(dv))
print(fmt % ("CUDA driver version", dv.value))
print("CUDA libraries:")
cudadrv.libs.test(sys.platform, print_paths=False)
except:
print(
"Error: Probing CUDA failed (device and driver present, runtime problem?)\n")
print("")
print("__ROC Information__")
roc_is_available = roc.is_available()
print(fmt % ("ROC available", roc_is_available))
toolchains = []
try:
libhlc.HLC()
toolchains.append('librocmlite library')
except:
pass
try:
cmd = hlc.CmdLine().check_tooling()
toolchains.append('ROC command line tools')
except:
pass
# if no ROC try and report why
if not roc_is_available:
from numba.roc.hsadrv.driver import hsa
try:
hsa.is_available
except BaseException as e:
msg = str(e)
else:
msg = 'No ROC toolchains found.'
print(fmt % ("Error initialising ROC due to", msg))
if toolchains:
print(fmt % ("Available Toolchains", ', '.join(toolchains)))
try:
# ROC might not be available due to lack of tool chain, but HSA
# agents may be listed
from numba.roc.hsadrv.driver import hsa, dgpu_count
decode = lambda x: x.decode('utf-8') if isinstance(x, bytes) else x
print("\nFound %s HSA Agents:" % len(hsa.agents))
for i, agent in enumerate(hsa.agents):
print('Agent id : %s' % i)
print(' vendor: %s' % decode(agent.vendor_name))
print(' name: %s' % decode(agent.name))
print(' type: %s' % agent.device)
print("")
_dgpus = []
for a in hsa.agents:
if a.is_component and a.device == 'GPU':
_dgpus.append(decode(a.name))
print(fmt % ("Found %s discrete GPU(s)" % dgpu_count(), \
', '.join(_dgpus)))
except Exception as e:
print("No HSA Agents found, encountered exception when searching:")
print(e)
print("")
print("__SVML Information__")
# replicate some SVML detection logic from numba.__init__ here.
# if SVML load fails in numba.__init__ the splitting of the logic
# here will help diagnosis of the underlying issue
have_svml_library = True
try:
if sys.platform.startswith('linux'):
llvmbind.load_library_permanently("libsvml.so")
elif sys.platform.startswith('darwin'):
llvmbind.load_library_permanently("libsvml.dylib")
elif sys.platform.startswith('win'):
llvmbind.load_library_permanently("svml_dispmd")
else:
have_svml_library = False
except:
have_svml_library = False
func = getattr(llvmbind.targets, "has_svml", None)
llvm_svml_patched = func() if func is not None else False
svml_operational = (config.USING_SVML and llvm_svml_patched \
and have_svml_library)
print(fmt % ("SVML state, config.USING_SVML", config.USING_SVML))
print(fmt % ("SVML library found and loaded", have_svml_library))
print(fmt % ("llvmlite using SVML patched LLVM", llvm_svml_patched))
print(fmt % ("SVML operational", svml_operational))
# Check which threading backends are available.
print("")
print("__Threading Layer Information__")
def parse_error(e, backend):
# parses a linux based error message, this is to provide feedback
# and hide user paths etc
try:
path, problem, symbol = [x.strip() for x in e.msg.split(':')]
extn_dso = os.path.split(path)[1]
if backend in extn_dso:
return "%s: %s" % (problem, symbol)
except BaseException:
pass
return "Unknown import problem."
try:
from numba.npyufunc import tbbpool
print(fmt % ("TBB Threading layer available", True))
except ImportError as e:
# might be a missing symbol due to e.g. tbb libraries missing
print(fmt % ("TBB Threading layer available", False))
print(fmt % ("+--> Disabled due to",
parse_error(e, 'tbbpool')))
try:
from numba.npyufunc import omppool
print(fmt % ("OpenMP Threading layer available", True))
except ImportError as e:
print(fmt % ("OpenMP Threading layer available", False))
print(fmt % ("+--> Disabled due to",
parse_error(e, 'omppool')))
try:
from numba.npyufunc import workqueue
print(fmt % ("Workqueue Threading layer available", True))
except ImportError as e:
print(fmt % ("Workqueue Threading layer available", False))
print(fmt % ("+--> Disabled due to",
parse_error(e, 'workqueue')))
# look for numba env vars that are set
print("")
print("__Numba Environment Variable Information__")
_envvar_found = False
for k, v in os.environ.items():
if k.startswith('NUMBA_'):
print(fmt % (k, v))
_envvar_found = True
if not _envvar_found:
print("None set.")
# Look for conda and conda information
print("")
print("__Conda Information__")
cmd = ["conda", "info", "--json"]
try:
conda_out = check_output(cmd)
except Exception as e:
print(
"Conda not present/not working.\nError was %s\n" % e)
else:
data = ''.join(conda_out.decode("utf-8").splitlines())
jsond = json.loads(data)
keys = ['conda_build_version',
'conda_env_version',
'platform',
'python_version',
'root_writable']
for k in keys:
try:
print(fmt % (k, jsond[k]))
except KeyError:
pass
# get info about current environment
cmd = ["conda", "list"]
try:
conda_out = check_output(cmd)
except CalledProcessError as e:
print("Error: Conda command failed. Error was %s\n" % e.output)
else:
print("")
print("__Current Conda Env__")
data = conda_out.decode("utf-8").splitlines()
for k in data:
if k[0] != '#': # don't show where the env is, personal data
print(k)
print("-" * 80)
except Exception as e:
print("Error: The system reporting tool has failed unexpectedly.")
print("Exception was:")
print(e)
finally:
print(
"%s" %
"If requested, please copy and paste the information between\n"
"the dashed (----) lines, or from a given specific section as\n"
"appropriate.\n\n"
"=============================================================\n"
"IMPORTANT: Please ensure that you are happy with sharing the\n"
"contents of the information present, any information that you\n"
"wish to keep private you should remove before sharing.\n"
"=============================================================\n")
def get_sys_info():
# delay these imports until now as they are only needed in this
# function which then exits.
import platform
import json
import multiprocessing
from numba import config
from numba import cuda as cu
from numba.cuda import cudadrv
from numba.cuda.cudadrv.driver import driver as cudriver
from numba import roc
from numba.roc.hlc import hlc, libhlc
import textwrap as tw
import ctypes as ct
import llvmlite.binding as llvmbind
import locale
from datetime import datetime
from itertools import chain
from subprocess import check_output, CalledProcessError
try:
fmt = "%-45s : %-s"
print("-" * 80)
print("__Time Stamp__")
print(datetime.utcnow())
print("")
print("__Hardware Information__")
system_name = platform.system()
print(fmt % ("Machine", platform.machine()))
print(fmt % ("CPU Name", llvmbind.get_host_cpu_name()))
if system_name == 'Linux':
strmatch = 'Cpus_allowed'
try:
loc = '/proc/self/status'
with open(loc, 'rt') as f:
proc_stat = f.read().splitlines()
for x in proc_stat:
if x.startswith(strmatch):
if x.startswith('%s:' % strmatch):
hexnum = '0x%s' % x.split(':')[1].strip()
acc_cpus = int(hexnum, 16)
_n = str(bin(acc_cpus).count('1'))
print(fmt %
("Number of accessible CPU cores", _n))
elif x.startswith('%s_list:' % strmatch):
_a = x.split(':')[1].strip()
print(fmt %
("Listed accessible CPUs cores", _a))
except Exception:
print(fmt % ("CPU count", multiprocessing.cpu_count()))
# See if CFS is in place
# https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
try:
def scrape_lines(loc):
with open(loc, 'rt') as f:
return f.read().splitlines()
loc = '/sys/fs/cgroup/cpuacct/cpu.cfs_period_us'
cfs_period = int(scrape_lines(loc)[0])
loc = '/sys/fs/cgroup/cpuacct/cpu.cfs_quota_us'
cfs_quota = int(scrape_lines(loc)[0])
if cfs_quota == -1:
print(fmt % ("CFS restrictions", "None"))
else:
runtime_amount = float(cfs_quota) / float(cfs_period)
print(fmt % ("CFS restrictions (CPUs worth of runtime)",
runtime_amount))
except Exception:
print(fmt % ("CFS restrictions", 'Information not available'))
else:
print(fmt % ("CPU count", multiprocessing.cpu_count()))
try:
featuremap = llvmbind.get_host_cpu_features()
except RuntimeError:
print(fmt % ("CPU Features", "NA"))
else:
features = sorted(
[key for key, value in featuremap.items() if value])
cpu_feat = tw.fill(' '.join(features), 80)
print(fmt % ("CPU Features", ""))
print(cpu_feat)
print("")
print("__OS Information__")
print(fmt % ("Platform", platform.platform(aliased=True)))
print(fmt % ("Release", platform.release()))
print(fmt % ("System Name", system_name))
print(fmt % ("Version", platform.version()))
try:
if system_name == 'Linux':
info = platform.linux_distribution()
elif system_name == 'Windows':
info = platform.win32_ver()
elif system_name == 'Darwin':
info = platform.mac_ver()
else:
raise RuntimeError("Unknown system.")
buf = ''.join([
x if x != '' else ' ' for x in list(chain.from_iterable(info))
])
print(fmt % ("OS specific info", buf))
if system_name == 'Linux':
print(fmt % ("glibc info", ' '.join(platform.libc_ver())))
except:
print("Error: System name incorrectly identified or unknown.")
print("")
print("__Python Information__")
print(fmt % ("Python Compiler", platform.python_compiler()))
print(fmt %
("Python Implementation", platform.python_implementation()))
print(fmt % ("Python Version", platform.python_version()))
lcl = []
try:
for x in locale.getdefaultlocale():
if x is not None:
lcl.append(x)
except Exception as e:
lcl.append(str(e))
print(fmt % ("Python Locale ", ' '.join(lcl)))
print("")
print("__LLVM information__")
print(fmt % ("LLVM version", '.'.join(
[str(k) for k in llvmbind.llvm_version_info])))
print("")
print("__CUDA Information__")
# Look for GPUs
try:
cu.list_devices()[0] # will a device initialise?
except Exception as e:
msg_not_found = "CUDA driver library cannot be found"
msg_disabled_by_user = "******"
msg_end = " or no CUDA enabled devices are present."
msg_generic_problem = "Error: CUDA device intialisation problem."
msg = getattr(e, 'msg', None)
if msg is not None:
if msg_not_found in msg:
err_msg = msg_not_found + msg_end
elif msg_disabled_by_user in msg:
err_msg = msg_disabled_by_user + msg_end
else:
err_msg = msg_generic_problem + " Message:" + msg
else:
err_msg = msg_generic_problem + " " + str(e)
# Best effort error report
print("%s\nError class: %s" % (err_msg, str(type(e))))
else:
try:
cu.detect()
dv = ct.c_int(0)
cudriver.cuDriverGetVersion(ct.byref(dv))
print(fmt % ("CUDA driver version", dv.value))
print("CUDA libraries:")
cudadrv.libs.test(sys.platform, print_paths=False)
except:
print(
"Error: Probing CUDA failed (device and driver present, runtime problem?)\n"
)
print("")
print("__ROC Information__")
roc_is_available = roc.is_available()
print(fmt % ("ROC available", roc_is_available))
toolchains = []
try:
libhlc.HLC()
toolchains.append('librocmlite library')
except:
pass
try:
cmd = hlc.CmdLine().check_tooling()
toolchains.append('ROC command line tools')
except:
pass
# if no ROC try and report why
if not roc_is_available:
from numba.roc.hsadrv.driver import hsa
try:
hsa.is_available
except Exception as e:
msg = str(e)
else:
msg = 'No ROC toolchains found.'
print(fmt % ("Error initialising ROC due to", msg))
if toolchains:
print(fmt % ("Available Toolchains", ', '.join(toolchains)))
try:
# ROC might not be available due to lack of tool chain, but HSA
# agents may be listed
from numba.roc.hsadrv.driver import hsa, dgpu_count
decode = lambda x: x.decode('utf-8') if isinstance(x, bytes) else x
print("\nFound %s HSA Agents:" % len(hsa.agents))
for i, agent in enumerate(hsa.agents):
print('Agent id : %s' % i)
print(' vendor: %s' % decode(agent.vendor_name))
print(' name: %s' % decode(agent.name))
print(' type: %s' % agent.device)
print("")
_dgpus = []
for a in hsa.agents:
if a.is_component and a.device == 'GPU':
_dgpus.append(decode(a.name))
print(fmt % ("Found %s discrete GPU(s)" % dgpu_count(), \
', '.join(_dgpus)))
except Exception as e:
print("No HSA Agents found, encountered exception when searching:")
print(e)
print("")
print("__SVML Information__")
# replicate some SVML detection logic from numba.__init__ here.
# if SVML load fails in numba.__init__ the splitting of the logic
# here will help diagnosis of the underlying issue
have_svml_library = True
try:
if sys.platform.startswith('linux'):
llvmbind.load_library_permanently("libsvml.so")
elif sys.platform.startswith('darwin'):
llvmbind.load_library_permanently("libsvml.dylib")
elif sys.platform.startswith('win'):
llvmbind.load_library_permanently("svml_dispmd")
else:
have_svml_library = False
except:
have_svml_library = False
func = getattr(llvmbind.targets, "has_svml", None)
llvm_svml_patched = func() if func is not None else False
svml_operational = (config.USING_SVML and llvm_svml_patched \
and have_svml_library)
print(fmt % ("SVML state, config.USING_SVML", config.USING_SVML))
print(fmt % ("SVML library found and loaded", have_svml_library))
print(fmt % ("llvmlite using SVML patched LLVM", llvm_svml_patched))
print(fmt % ("SVML operational", svml_operational))
# Check which threading backends are available.
print("")
print("__Threading Layer Information__")
def parse_error(e, backend):
# parses a linux based error message, this is to provide feedback
# and hide user paths etc
try:
path, problem, symbol = [x.strip() for x in e.msg.split(':')]
extn_dso = os.path.split(path)[1]
if backend in extn_dso:
return "%s: %s" % (problem, symbol)
except Exception:
pass
return "Unknown import problem."
try:
from numba.npyufunc import tbbpool
print(fmt % ("TBB Threading layer available", True))
except ImportError as e:
# might be a missing symbol due to e.g. tbb libraries missing
print(fmt % ("TBB Threading layer available", False))
print(fmt % ("+--> Disabled due to", parse_error(e, 'tbbpool')))
try:
from numba.npyufunc import omppool
print(fmt % ("OpenMP Threading layer available", True))
except ImportError as e:
print(fmt % ("OpenMP Threading layer available", False))
print(fmt % ("+--> Disabled due to", parse_error(e, 'omppool')))
try:
from numba.npyufunc import workqueue
print(fmt % ("Workqueue Threading layer available", True))
except ImportError as e:
print(fmt % ("Workqueue Threading layer available", False))
print(fmt % ("+--> Disabled due to", parse_error(e, 'workqueue')))
# look for numba env vars that are set
print("")
print("__Numba Environment Variable Information__")
_envvar_found = False
for k, v in os.environ.items():
if k.startswith('NUMBA_'):
print(fmt % (k, v))
_envvar_found = True
if not _envvar_found:
print("None set.")
# Look for conda and conda information
print("")
print("__Conda Information__")
cmd = ["conda", "info", "--json"]
try:
conda_out = check_output(cmd)
except Exception as e:
print("Conda not present/not working.\nError was %s\n" % e)
else:
data = ''.join(conda_out.decode("utf-8").splitlines())
jsond = json.loads(data)
keys = [
'conda_build_version', 'conda_env_version', 'platform',
'python_version', 'root_writable'
]
for k in keys:
try:
print(fmt % (k, jsond[k]))
except KeyError:
pass
# get info about current environment
cmd = ["conda", "list"]
try:
conda_out = check_output(cmd)
except CalledProcessError as e:
print("Error: Conda command failed. Error was %s\n" % e.output)
else:
print("")
print("__Current Conda Env__")
data = conda_out.decode("utf-8").splitlines()
for k in data:
if k[0] != '#': # don't show where the env is, personal data
print(k)
print("-" * 80)
except Exception as e:
print("Error: The system reporting tool has failed unexpectedly.")
print("Exception was:")
print(e)
finally:
print(
"%s" %
"If requested, please copy and paste the information between\n"
"the dashed (----) lines, or from a given specific section as\n"
"appropriate.\n\n"
"=============================================================\n"
"IMPORTANT: Please ensure that you are happy with sharing the\n"
"contents of the information present, any information that you\n"
"wish to keep private you should remove before sharing.\n"
"=============================================================\n")
def native_lowering_stage(targetctx, library, interp, typemap, restype,
calltypes, flags, metadata):
global LLVM_FILES
global FIRST_DEC_RUN
global func_cache
# Lowering
fndesc = funcdesc.PythonFunctionDescriptor.from_specialized_function(
interp, typemap, restype, calltypes, mangler=targetctx.mangler,
inline=flags.forceinline, noalias=flags.noalias)
with targetctx.push_code_library(library):
lower = lowering.Lower(targetctx, library, fndesc, interp,
metadata=metadata)
lower.lower()
if not flags.no_cpython_wrapper:
lower.create_cpython_wrapper(flags.release_gil)
env = lower.env
call_helper = lower.call_helper
del lower
if flags.no_compile:
return _LowerResult(fndesc, call_helper, cfunc=None, env=env)
else:
# Prepare for execution
functions = [x.name for x in library.get_defined_functions()]
is_kernel_compute = "_kernel_compute" in functions[0]
is_kernel_supply = "_kernel_supply" in functions[0]
is_tile_launcher = "tile_launcher" in functions[0]
call_decpp = None
if (DEC_Options.is_base_mode()):
call_decpp = is_kernel_compute or is_tile_launcher
if (DEC_Options.is_decoupled_mode()):
call_decpp = is_kernel_compute or is_kernel_supply or is_tile_launcher
assert(call_decpp is not None)
if not call_decpp:
pipe_line_print("")
pipe_line_print("-----")
pipe_line_print("DEC++ Pipeline compilation output")
pipe_line_print("")
pipe_line_print(functions)
for l in library._linking_libraries:
LINKING_LIBS_CACHES.append(l)
LIBS_TO_SRC[library] = library.get_llvm_str()
pipe_line_print("-----")
pipe_line_print("")
if call_decpp:
#create_fresh_dir(DECADES_OUTDIR)
if FIRST_DEC_RUN:
ll.load_library_permanently(os.path.join(DEC_options["pythia_home"],"tools", "tracer.so"))
ll.load_library_permanently(DEC_options["openmp_lib"])
ll.load_library_permanently(os.path.join(BUILD_ROOT, "./utils/DecoupleServer/libproduce_consume-shared.so"))
#pdb.set_trace()
ll.load_library_permanently(os.path.join(BUILD_ROOT, "./utils/DECLib/libDECADES-numba.so"))
pipe_line_print("")
pipe_line_print("-----")
pipe_line_print("DEC++ Pipeline compilation output")
pipe_line_print("")
functions = [x for x in library.get_defined_functions()]
pipe_line_print("identified kernel:")
pipe_line_print(functions[0].name)
pipe_line_print("")
def rec_mk_llvm_from_libs(lib):
global LLVM_FILES
if lib in func_cache:
return
else:
if lib in LIBS_TO_SRC:
to_print = LIBS_TO_SRC[lib]
else:
to_print = lib.get_llvm_str()
llvm_output_file = str(LLVM_FILES) + "_" + DEC_options["llvm_outfile"]
fh = open(os.path.join(DECADES_OUTDIR,llvm_output_file), 'w')
fh.write(to_print)
fh.close()
LLVM_FILES += 1
func_cache.append(lib)
for l in lib._linking_libraries:
rec_mk_llvm_from_libs(l)
rec_mk_llvm_from_libs(library)
for kernel in TO_COMPILE:
rec_mk_llvm_from_libs(kernel)
TO_COMPILE.append(library)
for l in library._linking_libraries:
LINKING_LIBS_CACHES.append(l)
cmd = DEC_Options.get_decpp_line()
#cmd = "DEC++ -fn 1 --target simulator -spp ~/pythia_git/pythia/pass/preproc.sh " + DECADES_OUTDIR
print("\nexecuting DEC++:")
print(cmd + "\n")
ret = os.system(cmd)
if ret != 0:
print("")
print("Error executing DEC++")
exit(0)
assert(os.path.exists(os.path.join(DECADES_BASE, DECpp_output)))
cmd = "sed -e s/immarg//g -i " + os.path.join(DECADES_BASE, DECpp_output)
pipe_line_print(cmd)
os.system(cmd)
fh = open(os.path.join(DECADES_BASE, DECpp_output))
llvm_src = fh.read()
fh.close()
reg_labels = re.findall(r"^\d+:\s*; preds =.*$", llvm_src, re.MULTILINE)
for rl in reg_labels:
new_line = "; <label>:" + rl
llvm_src = llvm_src.replace(rl, new_line)
codegen = library.codegen
library2 = codegen.create_library('DEC++')
#from pathlib import Path
#contents = Path(os.path.join(DECADES_BASE, DECpp_output)).read_text()
new_module = ll.parse_assembly(llvm_src)
library2.add_llvm_module(new_module)
library = library2
#pdb.set_trace()
#library._linking_libraries = linking_libs_cache
LLVM_FILES += 1
FIRST_DEC_RUN = False
cfunc = targetctx.get_executable(library, fndesc, env)
# Insert native function for use by other jitted-functions.
# We also register its library to allow for inlining.
targetctx.insert_user_function(cfunc, fndesc, [library])
return _LowerResult(fndesc, call_helper, cfunc=cfunc, env=env)
def link_library(self, file):
llvm.load_library_permanently(filename=file)
def get_sysinfo():
# Gather the information that shouldn't raise exceptions
sys_info = {
_start: datetime.now(),
_start_utc: datetime.utcnow(),
_machine: platform.machine(),
_cpu_name: llvmbind.get_host_cpu_name(),
_cpu_count: multiprocessing.cpu_count(),
_platform_name: platform.platform(aliased=True),
_platform_release: platform.release(),
_os_name: platform.system(),
_os_version: platform.version(),
_python_comp: platform.python_compiler(),
_python_impl: platform.python_implementation(),
_python_version: platform.python_version(),
_numba_env_vars: {k: v for (k, v) in os.environ.items()
if k.startswith('NUMBA_')},
_numba_version: version_number,
_llvm_version: '.'.join(str(i) for i in llvmbind.llvm_version_info),
_llvmlite_version: llvmlite_version,
_roc_available: roc.is_available(),
_psutil: _psutil_import,
}
# CPU features
try:
feature_map = llvmbind.get_host_cpu_features()
except RuntimeError as e:
_error_log.append(f'Error (CPU features): {e}')
else:
features = sorted([key for key, value in feature_map.items() if value])
sys_info[_cpu_features] = ' '.join(features)
# Python locale
# On MacOSX, getdefaultlocale can raise. Check again if Py > 3.7.5
try:
# If $LANG is unset, getdefaultlocale() can return (None, None), make
# sure we can encode this as strings by casting explicitly.
sys_info[_python_locale] = '.'.join([str(i) for i in
locale.getdefaultlocale()])
except Exception as e:
_error_log.append(f'Error (locale): {e}')
# CUDA information
try:
cu.list_devices()[0] # will a device initialise?
except Exception as e:
sys_info[_cu_dev_init] = False
msg_not_found = "CUDA driver library cannot be found"
msg_disabled_by_user = "******"
msg_end = " or no CUDA enabled devices are present."
msg_generic_problem = "CUDA device intialisation problem."
msg = getattr(e, 'msg', None)
if msg is not None:
if msg_not_found in msg:
err_msg = msg_not_found + msg_end
elif msg_disabled_by_user in msg:
err_msg = msg_disabled_by_user + msg_end
else:
err_msg = msg_generic_problem + " Message:" + msg
else:
err_msg = msg_generic_problem + " " + str(e)
# Best effort error report
_warning_log.append("Warning (cuda): %s\nException class: %s" %
(err_msg, str(type(e))))
else:
try:
sys_info[_cu_dev_init] = True
output = StringIO()
with redirect_stdout(output):
cu.detect()
sys_info[_cu_detect_out] = output.getvalue()
output.close()
dv = ctypes.c_int(0)
cudriver.cuDriverGetVersion(ctypes.byref(dv))
sys_info[_cu_drv_ver] = dv.value
rtver = ctypes.c_int(0)
curuntime.cudaRuntimeGetVersion(ctypes.byref(rtver))
sys_info[_cu_rt_ver] = rtver.value
output = StringIO()
with redirect_stdout(output):
cudadrv.libs.test(sys.platform, print_paths=False)
sys_info[_cu_lib_test] = output.getvalue()
output.close()
except Exception as e:
_warning_log.append(
"Warning (cuda): Probing CUDA failed "
"(device and driver present, runtime problem?)\n"
f"(cuda) {type(e)}: {e}")
# ROC information
# If no ROC try and report why
if not sys_info[_roc_available]:
from numba.roc.hsadrv.driver import hsa
try:
hsa.is_available
except Exception as e:
msg = str(e)
else:
msg = 'No ROC toolchains found.'
_warning_log.append(f"Warning (roc): Error initialising ROC: {msg}")
toolchains = []
try:
libhlc.HLC()
toolchains.append('librocmlite library')
except Exception:
pass
try:
cmd = hlc.CmdLine().check_tooling()
toolchains.append('ROC command line tools')
except Exception:
pass
sys_info[_roc_toolchains] = toolchains
try:
# ROC might not be available due to lack of tool chain, but HSA
# agents may be listed
from numba.roc.hsadrv.driver import hsa, dgpu_count
def decode(x):
return x.decode('utf-8') if isinstance(x, bytes) else x
sys_info[_hsa_agents_count] = len(hsa.agents)
agents = []
for i, agent in enumerate(hsa.agents):
agents.append({
'Agent id': i,
'Vendor': decode(agent.vendor_name),
'Name': decode(agent.name),
'Type': agent.device,
})
sys_info[_hsa_agents] = agents
_dgpus = []
for a in hsa.agents:
if a.is_component and a.device == 'GPU':
_dgpus.append(decode(a.name))
sys_info[_hsa_gpus_count] = dgpu_count()
sys_info[_hsa_gpus] = ', '.join(_dgpus)
except Exception as e:
_warning_log.append(
"Warning (roc): No HSA Agents found, "
f"encountered exception when searching: {e}")
# SVML information
# Replicate some SVML detection logic from numba.__init__ here.
# If SVML load fails in numba.__init__ the splitting of the logic
# here will help diagnosing the underlying issue.
svml_lib_loaded = True
try:
if sys.platform.startswith('linux'):
llvmbind.load_library_permanently("libsvml.so")
elif sys.platform.startswith('darwin'):
llvmbind.load_library_permanently("libsvml.dylib")
elif sys.platform.startswith('win'):
llvmbind.load_library_permanently("svml_dispmd")
else:
svml_lib_loaded = False
except Exception:
svml_lib_loaded = False
func = getattr(llvmbind.targets, "has_svml", None)
sys_info[_llvm_svml_patched] = func() if func else False
sys_info[_svml_state] = config.USING_SVML
sys_info[_svml_loaded] = svml_lib_loaded
sys_info[_svml_operational] = all((
sys_info[_svml_state],
sys_info[_svml_loaded],
sys_info[_llvm_svml_patched],
))
# Check which threading backends are available.
def parse_error(e, backend):
# parses a linux based error message, this is to provide feedback
# and hide user paths etc
try:
path, problem, symbol = [x.strip() for x in e.msg.split(':')]
extn_dso = os.path.split(path)[1]
if backend in extn_dso:
return "%s: %s" % (problem, symbol)
except Exception:
pass
return "Unknown import problem."
try:
from numba.np.ufunc import tbbpool # NOQA
sys_info[_tbb_thread] = True
except ImportError as e:
# might be a missing symbol due to e.g. tbb libraries missing
sys_info[_tbb_thread] = False
sys_info[_tbb_error] = parse_error(e, 'tbbpool')
try:
from numba.np.ufunc import omppool
sys_info[_openmp_thread] = True
sys_info[_openmp_vendor] = omppool.openmp_vendor
except ImportError as e:
sys_info[_openmp_thread] = False
sys_info[_openmp_error] = parse_error(e, 'omppool')
try:
from numba.np.ufunc import workqueue # NOQA
sys_info[_wkq_thread] = True
except ImportError as e:
sys_info[_wkq_thread] = True
sys_info[_wkq_error] = parse_error(e, 'workqueue')
# Look for conda and installed packages information
cmd = ('conda', 'info', '--json')
try:
conda_out = check_output(cmd)
except Exception as e:
_warning_log.append(f'Warning: Conda not available.\n Error was {e}\n')
# Conda is not available, try pip list to list installed packages
cmd = (sys.executable, '-m', 'pip', 'list')
try:
reqs = check_output(cmd)
except Exception as e:
_error_log.append(f'Error (pip): {e}')
else:
sys_info[_inst_pkg] = reqs.decode().splitlines()
else:
jsond = json.loads(conda_out.decode())
keys = {
'conda_build_version': _conda_build_ver,
'conda_env_version': _conda_env_ver,
'platform': _conda_platform,
'python_version': _conda_python_ver,
'root_writable': _conda_root_writable,
}
for conda_k, sysinfo_k in keys.items():
sys_info[sysinfo_k] = jsond.get(conda_k, 'N/A')
# Get info about packages in current environment
cmd = ('conda', 'list')
try:
conda_out = check_output(cmd)
except CalledProcessError as e:
_error_log.append(f'Error (conda): {e}')
else:
data = conda_out.decode().splitlines()
sys_info[_inst_pkg] = [l for l in data if not l.startswith('#')]
sys_info.update(get_os_spec_info(sys_info[_os_name]))
sys_info[_errors] = _error_log
sys_info[_warnings] = _warning_log
sys_info[_runtime] = (datetime.now() - sys_info[_start]).total_seconds()
return sys_info
def timeout():
print("timed out!")
exit(0)
if __name__ == "__main__":
# cd to where all the good stuff is located
os.chdir(sys.path[0])
# alarm
signal.signal(signal.SIGALRM, timeout)
signal.alarm(300)
# initialize llvm backend
llvm.initialize()
llvm.initialize_native_target()
llvm.initialize_native_asmprinter()
llvm.load_library_permanently("./runtime.so")
# initialize libruntime
lruntime = CDLL("./runtime.so")
lruntime.alloc_data.restype = c_long
# compile banner printing program
banner()
while True:
code = input(">>> ")
l, h = compile(bfProgram(code))
execute(l, h)
parser = SLRParser("new_grammar.txt", '../testInputs/' + name)
syn_tree = parser.parse()[0]
syn_tree.genCases(parser.grammar)
sym_table = SymbolTable()
named_sym_table = NamedSymbolTable()
valid = errorCheck(syn_tree, sym_table, named_sym_table,
parser.lexer.infileLines)
ctx = ir.global_context
llvm_custom_types = create_llvm_types(named_sym_table, ctx)
generated_ir = codegen(syn_tree, named_sym_table, {
'mod': module,
'llvm_custom_types': llvm_custom_types,
'ctx': ctx,
})
print(module)
binding.initialize()
binding.initialize_native_target()
binding.initialize_native_asmprinter() # yes, even this one
binding.load_library_permanently('../runtime.so')
binding.parse_assembly(str(module), None)
engine = create_execution_engine()
mod = compile_ir(engine, str(module))
t_fun = CFUNCTYPE(c_int64, c_int64)
fact = t_fun(engine.get_function_address('global.fun1'))
print(fact(5, 7))
def load_library(self, filename: str) -> None:
binding.load_library_permanently(filename)
from __future__ import print_function
from ctypes import CFUNCTYPE, c_int
import llvmlite.binding as llvm
import sys
# All these initializations are required for code generation!
llvm.initialize()
llvm.initialize_native_target()
llvm.initialize_native_asmprinter() # yes, even this one
llvm.load_library_permanently("C:\\Windows\\System32\\msvcrt.dll")
def create_execution_engine():
"""
Create an ExecutionEngine suitable for JIT code generation on
the host CPU. The engine is reusable for an arbitrary number of
modules.
"""
# Create a target machine representing the host
target = llvm.Target.from_default_triple()
target_machine = target.create_target_machine()
# And an execution engine with an empty backing module
backing_mod = llvm.parse_assembly("")
engine = llvm.create_mcjit_compiler(backing_mod, target_machine)
return engine
def get_sysinfo():
# Gather the information that shouldn't raise exceptions
sys_info = {
_start: datetime.now(),
_start_utc: datetime.utcnow(),
_machine: platform.machine(),
_cpu_name: llvmbind.get_host_cpu_name(),
_cpu_count: multiprocessing.cpu_count(),
_platform_name: platform.platform(aliased=True),
_platform_release: platform.release(),
_os_name: platform.system(),
_os_version: platform.version(),
_python_comp: platform.python_compiler(),
_python_impl: platform.python_implementation(),
_python_version: platform.python_version(),
_numba_env_vars:
{k: v
for (k, v) in os.environ.items() if k.startswith('NUMBA_')},
_numba_version: version_number,
_llvm_version: '.'.join(str(i) for i in llvmbind.llvm_version_info),
_llvmlite_version: llvmlite_version,
_psutil: _psutil_import,
}
# CPU features
try:
feature_map = llvmbind.get_host_cpu_features()
except RuntimeError as e:
_error_log.append(f'Error (CPU features): {e}')
else:
features = sorted([key for key, value in feature_map.items() if value])
sys_info[_cpu_features] = ' '.join(features)
# Python locale
# On MacOSX, getdefaultlocale can raise. Check again if Py > 3.7.5
try:
# If $LANG is unset, getdefaultlocale() can return (None, None), make
# sure we can encode this as strings by casting explicitly.
sys_info[_python_locale] = '.'.join(
[str(i) for i in locale.getdefaultlocale()])
except Exception as e:
_error_log.append(f'Error (locale): {e}')
# CUDA information
try:
cu.list_devices()[0] # will a device initialise?
except Exception as e:
sys_info[_cu_dev_init] = False
msg_not_found = "CUDA driver library cannot be found"
msg_disabled_by_user = "******"
msg_end = " or no CUDA enabled devices are present."
msg_generic_problem = "CUDA device initialisation problem."
msg = getattr(e, 'msg', None)
if msg is not None:
if msg_not_found in msg:
err_msg = msg_not_found + msg_end
elif msg_disabled_by_user in msg:
err_msg = msg_disabled_by_user + msg_end
else:
err_msg = msg_generic_problem + " Message:" + msg
else:
err_msg = msg_generic_problem + " " + str(e)
# Best effort error report
_warning_log.append("Warning (cuda): %s\nException class: %s" %
(err_msg, str(type(e))))
else:
try:
sys_info[_cu_dev_init] = True
output = StringIO()
with redirect_stdout(output):
cu.detect()
sys_info[_cu_detect_out] = output.getvalue()
output.close()
sys_info[_cu_drv_ver] = '%s.%s' % cudriver.get_version()
sys_info[_cu_rt_ver] = '%s.%s' % curuntime.get_version()
output = StringIO()
with redirect_stdout(output):
cudadrv.libs.test(sys.platform, print_paths=False)
sys_info[_cu_lib_test] = output.getvalue()
output.close()
try:
from cuda import cuda # noqa: F401
nvidia_bindings_available = True
except ImportError:
nvidia_bindings_available = False
sys_info[_cu_nvidia_bindings] = nvidia_bindings_available
nv_binding_used = bool(cudadrv.driver.USE_NV_BINDING)
sys_info[_cu_nvidia_bindings_used] = nv_binding_used
except Exception as e:
_warning_log.append(
"Warning (cuda): Probing CUDA failed "
"(device and driver present, runtime problem?)\n"
f"(cuda) {type(e)}: {e}")
# NumPy information
sys_info[_numpy_version] = np.version.full_version
try:
# NOTE: These consts were added in NumPy 1.20
from numpy.core._multiarray_umath import (
__cpu_features__,
__cpu_dispatch__,
__cpu_baseline__,
)
except ImportError:
sys_info[_numpy_AVX512_SKX_detected] = False
else:
feat_filtered = [k for k, v in __cpu_features__.items() if v]
sys_info[_numpy_supported_simd_features] = feat_filtered
sys_info[_numpy_supported_simd_dispatch] = __cpu_dispatch__
sys_info[_numpy_supported_simd_baseline] = __cpu_baseline__
sys_info[_numpy_AVX512_SKX_detected] = \
__cpu_features__.get("AVX512_SKX", False)
# SVML information
# Replicate some SVML detection logic from numba.__init__ here.
# If SVML load fails in numba.__init__ the splitting of the logic
# here will help diagnosing the underlying issue.
svml_lib_loaded = True
try:
if sys.platform.startswith('linux'):
llvmbind.load_library_permanently("libsvml.so")
elif sys.platform.startswith('darwin'):
llvmbind.load_library_permanently("libsvml.dylib")
elif sys.platform.startswith('win'):
llvmbind.load_library_permanently("svml_dispmd")
else:
svml_lib_loaded = False
except Exception:
svml_lib_loaded = False
func = getattr(llvmbind.targets, "has_svml", None)
sys_info[_llvm_svml_patched] = func() if func else False
sys_info[_svml_state] = config.USING_SVML
sys_info[_svml_loaded] = svml_lib_loaded
sys_info[_svml_operational] = all((
sys_info[_svml_state],
sys_info[_svml_loaded],
sys_info[_llvm_svml_patched],
))
# Check which threading backends are available.
def parse_error(e, backend):
# parses a linux based error message, this is to provide feedback
# and hide user paths etc
try:
path, problem, symbol = [x.strip() for x in e.msg.split(':')]
extn_dso = os.path.split(path)[1]
if backend in extn_dso:
return "%s: %s" % (problem, symbol)
except Exception:
pass
return "Unknown import problem."
try:
# check import is ok, this means the DSO linkage is working
from numba.np.ufunc import tbbpool # NOQA
# check that the version is compatible, this is a check performed at
# runtime (well, compile time), it will also ImportError if there's
# a problem.
from numba.np.ufunc.parallel import _check_tbb_version_compatible
_check_tbb_version_compatible()
sys_info[_tbb_thread] = True
except ImportError as e:
# might be a missing symbol due to e.g. tbb libraries missing
sys_info[_tbb_thread] = False
sys_info[_tbb_error] = parse_error(e, 'tbbpool')
try:
from numba.np.ufunc import omppool
sys_info[_openmp_thread] = True
sys_info[_openmp_vendor] = omppool.openmp_vendor
except ImportError as e:
sys_info[_openmp_thread] = False
sys_info[_openmp_error] = parse_error(e, 'omppool')
try:
from numba.np.ufunc import workqueue # NOQA
sys_info[_wkq_thread] = True
except ImportError as e:
sys_info[_wkq_thread] = True
sys_info[_wkq_error] = parse_error(e, 'workqueue')
# Look for conda and installed packages information
cmd = ('conda', 'info', '--json')
try:
conda_out = check_output(cmd)
except Exception as e:
_warning_log.append(f'Warning: Conda not available.\n Error was {e}\n')
# Conda is not available, try pip list to list installed packages
cmd = (sys.executable, '-m', 'pip', 'list')
try:
reqs = check_output(cmd)
except Exception as e:
_error_log.append(f'Error (pip): {e}')
else:
sys_info[_inst_pkg] = reqs.decode().splitlines()
else:
jsond = json.loads(conda_out.decode())
keys = {
'conda_build_version': _conda_build_ver,
'conda_env_version': _conda_env_ver,
'platform': _conda_platform,
'python_version': _conda_python_ver,
'root_writable': _conda_root_writable,
}
for conda_k, sysinfo_k in keys.items():
sys_info[sysinfo_k] = jsond.get(conda_k, 'N/A')
# Get info about packages in current environment
cmd = ('conda', 'list')
try:
conda_out = check_output(cmd)
except CalledProcessError as e:
_error_log.append(f'Error (conda): {e}')
else:
data = conda_out.decode().splitlines()
sys_info[_inst_pkg] = [l for l in data if not l.startswith('#')]
sys_info.update(get_os_spec_info(sys_info[_os_name]))
sys_info[_errors] = _error_log
sys_info[_warnings] = _warning_log
sys_info[_runtime] = (datetime.now() - sys_info[_start]).total_seconds()
return sys_info
def test_bad_library(self):
with self.assertRaises(RuntimeError):
llvm.load_library_permanently("zzzasdkf;jasd;l")
import platform
import re
import subprocess
import sys
import unittest
from contextlib import contextmanager
from tempfile import mkstemp
from llvmlite import ir
from llvmlite import binding as llvm
from llvmlite.binding import ffi
from llvmlite.tests import TestCase
# arvm7l needs extra ABI symbols to link successfully
if platform.machine() == "armv7l":
llvm.load_library_permanently("libgcc_s.so.1")
def no_de_locale():
cur = locale.setlocale(locale.LC_ALL)
try:
locale.setlocale(locale.LC_ALL, "de_DE")
except locale.Error:
return True
else:
return False
finally:
locale.setlocale(locale.LC_ALL, cur)
asm_sum = r"""
EXTERNAL_LIBS = []
for extlib_pattern in ["SparseTensorUtils*.so", "RandomUtils*.so"]:
_CURRENT_MODULE_DIR = os.path.dirname(__file__)
SO_FILE_PATTERN = os.path.join(_CURRENT_MODULE_DIR, extlib_pattern)
SO_FILES = glob.glob(SO_FILE_PATTERN)
if len(SO_FILES) == 0:
# TODO this hard-codes the setup.py option and the location of setup.py
raise RuntimeError(
f"{SO_FILE_PATTERN} not found. This can typically be solved "
f'by running "python setup.py build_ext" from {os.path.dirname(_CURRENT_MODULE_DIR)}.'
)
elif len(SO_FILES) > 1:
raise RuntimeError(f"Multiple files matching {SO_FILE_PATTERN} found.")
[SO_FILE] = SO_FILES
llvm.load_library_permanently(
SO_FILE
) # TODO will this cause name collisions with other uses of llvmlite by third-party libraries?
EXTERNAL_LIBS.append(SO_FILE)
llvm.initialize()
llvm.initialize_native_target()
llvm.initialize_native_asmprinter()
# try to load libomp on Linux
SO_FILE_PATTERN = os.path.join(sys.prefix, "lib", "libomp.so")
SO_FILES = glob.glob(SO_FILE_PATTERN)
if len(SO_FILES) > 0:
print(f"Loading {SO_FILES[0]}", file=sys.stderr)
llvm.load_library_permanently(SO_FILES[0])
MLIR_FLOAT_ENUM_TO_NP_TYPE = {
mlir.astnodes.FloatTypeEnum.f16: np.float16,
def load_shared_library(name):
"""Find and load shared library."""
if (SOURCE_DIRECTORY / name).exists():
binding.load_library_permanently(str(SOURCE_DIRECTORY / name))
else:
binding.load_library_permanently(str(SYSTEM_SHARED / name))