예제 #1
0
def MultiGPUWrapper(photons_requested, max_photons_to_run, *args, **kwargs):
    data_out = {}
    children = []
    n_GPUs = len(cuda.list_devices())

    # I've decided to use the photon numbers as is - it means that the requests should be treated
    # as a per GPU basis. The calling function is reponsible for dividing if neccesary.
    # photons_req_per_device = np.floor(photons_requested / n_GPUs)
    # max_photons_per_device = np.floor(max_photons_to_run / n_GPUs)

    photons_req_per_device = photons_requested
    max_photons_per_device = max_photons_to_run

    for device_id, dev in enumerate(cuda.list_devices()):
        data_out[device_id] = [0, 0]
        t = threading.Thread(target=GPUWrapper,
                             args=(data_out, device_id, photons_req_per_device,
                                   max_photons_per_device, *args),
                             kwargs=kwargs)
        t.start()
        children.append(t)

    for t in children:
        t.join()

    data_ret = None
    for i in range(n_GPUs):
        if data_ret is None:
            data_ret = data_out[i][0]
            photon_counters = data_out[i][1]
        else:
            data_ret = np.concatenate((data_ret, data_out[i][0]), axis=0)
            photon_counters += data_out[i][1]

    return [data_ret, photon_counters]
예제 #2
0
    def _do_discovery(self, argv, Loader=None):
        """The discovery process is complicated by the fact that:

        * different test suites live under different directories
        * some test suites may not be available (CUDA)
        * some tests may have to be run serially, even in the presence of the '-m' flag."""

        from numba import cuda
        join = os.path.join
        loader = unittest.TestLoader() if Loader is None else Loader()
        topdir = os.path.abspath(join(os.path.dirname(__file__), '../..'))
        base_tests = loader.discover(join(topdir, 'numba/tests'), 'test*.py',
                                     topdir)
        cuda_tests = [
            loader.discover(join(topdir, 'numba/cuda/tests/nocuda'),
                            'test*.py', topdir)
        ]
        if cuda.is_available():
            gpus = cuda.list_devices()
            if gpus and gpus[0].compute_capability >= (2, 0):
                cuda_tests.append(
                    loader.discover(join(topdir, 'numba/cuda/tests/cudadrv'),
                                    'test*.py', topdir))
                cuda_tests.append(
                    loader.discover(join(topdir, 'numba/cuda/tests/cudapy'),
                                    'test*.py', topdir))
            else:
                print("skipped CUDA tests because GPU CC < 2.0")
        else:
            print("skipped CUDA tests")
        self.test = suite.TestSuite(tests=(base_tests,
                                           SerialSuite(cuda_tests)))
예제 #3
0
파일: testing.py 프로젝트: smtlify/numba
def test(**kwargs):
    """
    Run all tests under ``numba.tests``.

    kwargs
    ------
    - descriptions
    - verbosity
    - buffer
    - failfast
    - xmloutput [str]
        Path of XML output directory
    """
    from numba import cuda

    suite = discover_tests("numba.tests")
    ok = run_tests(suite, **kwargs).wasSuccessful()
    if ok:
        if cuda.is_available():
            gpus = cuda.list_devices()
            if gpus and gpus[0].compute_capability >= (2, 0):
                print("== Run CUDA tests ==")
                ok = cuda.test()
            else:
                print("== Skipped CUDA tests because GPU CC < 2.0 ==")
        else:
            print("== Skipped CUDA tests ==")

    return ok
예제 #4
0
def calculate_matrix_profile(column, seq_length):
    import stumpy

    try:
        # stumpy needs np float
        old_data = np.array(column, dtype=np.floating)
    except ValueError:
        raise Exception('Can\'t convert column to float')

    try:
        if cuda.is_available():
            gpu_device_ids = [device.id for device in cuda.list_devices()]
            mp = stumpy.gpu_stump(old_data,
                                  m=seq_length,
                                  ignore_trivial=False,
                                  device_id=gpu_device_ids)
        else:
            mp = stumpy.stump(old_data, m=seq_length, ignore_trivial=False)

    except TypeError as e:
        print('Type issue in stumpy:')
        raise e
    except ValueError as e:
        print('Seq_length issue in stumpy')
        raise e

    if pd.isnull(mp).any():
        raise Exception(
            'Matrix profile for the column contains NaN values. Try to increase the dataset size'
        )

    return mp
예제 #5
0
def test_parallel_gpu_stump_self_join(T_A, T_B):
    device_ids = [device.id for device in cuda.list_devices()]
    if len(T_B) > 10:
        m = 3
        zone = int(np.ceil(m / 4))
        left = np.array(
            [
                utils.naive_mass(Q, T_B, m, i, zone, True)
                for i, Q in enumerate(core.rolling_window(T_B, m))
            ],
            dtype=object,
        )
        right = gpu_stump(
            T_B,
            m,
            ignore_trivial=True,
            threads_per_block=THREADS_PER_BLOCK,
            device_id=device_ids,
        )
        utils.replace_inf(left)
        utils.replace_inf(right)
        npt.assert_almost_equal(left, right)

        right = gpu_stump(
            pd.Series(T_B),
            m,
            ignore_trivial=True,
            threads_per_block=THREADS_PER_BLOCK,
            device_id=device_ids,
        )
        utils.replace_inf(right)
        npt.assert_almost_equal(left, right)
예제 #6
0
def test_parallel_gpu_stump_A_B_join(T_A, T_B):
    device_ids = [device.id for device in cuda.list_devices()]
    if len(T_B) > 10:
        m = 3
        left = np.array(
            [utils.naive_mass(Q, T_A, m) for Q in core.rolling_window(T_B, m)],
            dtype=object,
        )
        right = gpu_stump(
            T_A,
            m,
            T_B,
            ignore_trivial=False,
            threads_per_block=THREADS_PER_BLOCK,
            device_id=device_ids,
        )
        utils.replace_inf(left)
        utils.replace_inf(right)
        npt.assert_almost_equal(left, right)

        right = gpu_stump(
            pd.Series(T_A),
            m,
            pd.Series(T_B),
            ignore_trivial=False,
            threads_per_block=THREADS_PER_BLOCK,
            device_id=device_ids,
        )
        utils.replace_inf(right)
        npt.assert_almost_equal(left, right)
예제 #7
0
def gpuWork(stuff):
    batch, batch_size, gpu_id, data_len, data, parser = stuff
    workers = multiprocessing.cpu_count() / len(cuda.list_devices())
    pool = ThreadPool(workers)
    cuda.select_device(gpu_id)

    start = batch * batch_size
    end = (batch + 1) * batch_size
    if end > data_len:
        end = data_len
    results = pool.map(
        randRotateAndTranslateWrap,
        zip([parser] * (end - start), data[start:end], [
            gpu_id,
        ] * len(data[start:end]), [True] * len(data[start:end])))

    images, energies = [], []
    for im, e in results:
        images.append(im)
        energies.append(e)

    images = np.array(images, dtype=np.float32)
    energies = np.array(energies, dtype=np.float32)
    pool.close()
    pool.join()
    return images, energies
예제 #8
0
def test_parallel_gpu_stump_A_B_join(T_A, T_B):
    device_ids = [device.id for device in cuda.list_devices()]
    if len(T_B) > 10:
        m = 3
        left = naive.stamp(T_A, m, T_B=T_B)
        right = gpu_stump(
            T_A,
            m,
            T_B,
            ignore_trivial=False,
            device_id=device_ids,
        )
        naive.replace_inf(left)
        naive.replace_inf(right)
        npt.assert_almost_equal(left, right)

        right = gpu_stump(
            pd.Series(T_A),
            m,
            pd.Series(T_B),
            ignore_trivial=False,
            device_id=device_ids,
        )
        naive.replace_inf(right)
        npt.assert_almost_equal(left, right)
예제 #9
0
    def __init__(self):
        # System
        plat = platform.uname()
        self.system = plat.system
        self.node = plat.node
        self.ip = self.get_ip()

        # GPU
        devices: cuda.cudadrv.devices._DeviceList = cuda.list_devices()
        try:
            self.num_devices = len(devices)
            self.devices = [Device(d) for d in devices]
        except cuda.CudaSupportError:
            self.num_devices = 0

        # CPU
        self.logical_cpus = psutil.cpu_count(logical=True)
        self.physical_cpus = psutil.cpu_count(logical=False)
        freq = psutil.cpu_freq(percpu=False)
        self.min_cpu_freq = freq.min
        self.max_cpu_freq = freq.max

        # Memory
        self.virtual_memory = Bytes(psutil.virtual_memory().total)
        self.swap_memory = Bytes(psutil.swap_memory().total)
예제 #10
0
def MultiGPUWrapper(photons_requested, max_photons_to_run,
                    muS, g,
                    source_type, source_param1, source_param2,
                    detector_params,
                    max_N, max_distance_from_det,
                    target_type, target_mask, target_gridsize,
                    z_target, z_bounded, z_range, ret_cols):
    data_out = {}
    children = []
    n_GPUs = len(cuda.list_devices())

    # I've decided to use the photon numbers as is - it means that the requests should be treated
    # as a per GPU basis. The calling function is reponsible for dividing if neccesary.
    # photons_req_per_device = np.floor(photons_requested / n_GPUs)
    # max_photons_per_device = np.floor(max_photons_to_run / n_GPUs)

    photons_req_per_device = photons_requested
    max_photons_per_device = max_photons_to_run

    for device_id, dev in enumerate(cuda.list_devices()):
        data_out[device_id] = [0,0]
        t = threading.Thread(target=GPUWrapper, args=(data_out, device_id,
                                                      photons_req_per_device, max_photons_per_device,
                                                      muS, g,
                                                      source_type, source_param1, source_param2,
                                                      detector_params,
                                                      max_N, max_distance_from_det,
                                                      target_type, target_mask, target_gridsize,
                                                      z_target, z_bounded, z_range, ret_cols))
        t.start()
        children.append(t)

    for t in children:
        t.join()

    data_ret = None
    for i in range(n_GPUs):
        if data_ret is None:
            data_ret = data_out[i][0]
            photon_counters = data_out[i][1]
        else:
            data_ret = np.concatenate((data_ret, data_out[i][0]), axis=0)
            photon_counters += data_out[i][1]

    return [data_ret, photon_counters]
예제 #11
0
def main():

    if cuda.is_available():
        dev_no = cuda.cudadrv.driver.Device(0).id
        print(cuda.list_devices())
        print(cuda.cudadrv.driver.Device(dev_no).compute_capability)
        print(cuda.cudadrv.driver.Device(dev_no).name)
    else:
        print("no GPU detected")
예제 #12
0
def main():
    children = []
    for cid, dev in enumerate(cuda.list_devices()):
        t = threading.Thread(target=device_controller, args=(cid, ))
        t.start()
        children.append(t)

    for t in children:
        t.join()
    print('ending gracefully')
예제 #13
0
def main():
    children = []
    for cid, dev in enumerate(cuda.list_devices()):
        t = threading.Thread(target=device_controller, args=(cid,))
        t.start()
        children.append(t)

    for t in children:
        t.join()

    print('ending gracefully')
예제 #14
0
def skip_cuda_tests():

    try:
        if cuda.is_available():
            gpus = cuda.list_devices()
            if gpus and gpus[0].compute_capability >= (2, 0):
                return False
            else:
                return True
        return True
    except CudaSupportError:
        return True
예제 #15
0
 def clear_context(self) -> None:
     try:
         print("Clearing Context")
         devices_list: List[cuda.cudadrv.devices.
                            _DeviceContextManager] = cuda.list_devices().lst
         for device in devices_list:
             print("GPU device id:{}".format(device.id))
             cuda.select_device(device.id)
             cuda.close()
             device.reset()
     except cuda.cudadrv.error.CudaSupportError as e:
         pass
     finally:
         print("Context Cleared")
예제 #16
0
def test_parallel_gpu_stump_self_join(T_A, T_B):
    device_ids = [device.id for device in cuda.list_devices()]
    if len(T_B) > 10:
        m = 3
        zone = int(np.ceil(m / 4))
        left = naive.stamp(T_B, m, exclusion_zone=zone)
        right = gpu_stump(T_B, m, ignore_trivial=True, device_id=device_ids,)
        naive.replace_inf(left)
        naive.replace_inf(right)
        npt.assert_almost_equal(left, right)

        right = gpu_stump(pd.Series(T_B), m, ignore_trivial=True, device_id=device_ids,)
        naive.replace_inf(right)
        npt.assert_almost_equal(left, right)
예제 #17
0
def test_parallel_gpu_aamp_self_join(T_A, T_B):
    device_ids = [device.id for device in cuda.list_devices()]
    if len(T_B) > 10:
        m = 3
        zone = int(np.ceil(m / 4))
        ref_mp = naive.aamp(T_B, m, exclusion_zone=zone)
        comp_mp = gpu_aamp(
            T_B,
            m,
            ignore_trivial=True,
            device_id=device_ids,
        )
        naive.replace_inf(ref_mp)
        naive.replace_inf(comp_mp)
        npt.assert_almost_equal(ref_mp, comp_mp)
예제 #18
0
def load_tests(loader, tests, pattern):
    suite = unittest.TestSuite()
    this_dir = dirname(__file__)
    suite.addTests(load_testsuite(loader, join(this_dir, "nocuda")))
    if cuda.is_available():
        suite.addTests(load_testsuite(loader, join(this_dir, "cudasim")))
        gpus = cuda.list_devices()
        if gpus and gpus[0].compute_capability >= (2, 0):
            suite.addTests(load_testsuite(loader, join(this_dir, "cudadrv")))
            suite.addTests(load_testsuite(loader, join(this_dir, "cudapy")))
        else:
            print("skipped CUDA tests because GPU CC < 2.0")
    else:
        print("skipped CUDA tests")
    return suite
예제 #19
0
def test_parallel_gpu_aamp_A_B_join(T_A, T_B):
    device_ids = [device.id for device in cuda.list_devices()]
    if len(T_B) > 10:
        m = 3
        ref_mp = naive.aamp(T_B, m, T_B=T_A)
        comp_mp = gpu_aamp(
            T_B,
            m,
            T_A,
            ignore_trivial=False,
            device_id=device_ids,
        )
        naive.replace_inf(ref_mp)
        naive.replace_inf(comp_mp)
        npt.assert_almost_equal(ref_mp, comp_mp)
예제 #20
0
파일: __init__.py 프로젝트: EGQM/numba
def load_tests(loader, tests, pattern):

    suite = SerialSuite()
    this_dir = dirname(__file__)
    suite.addTests(load_testsuite(loader, join(this_dir, 'nocuda')))
    if cuda.is_available():
        gpus = cuda.list_devices()
        if gpus and gpus[0].compute_capability >= (2, 0):
            suite.addTests(load_testsuite(loader, join(this_dir, 'cudadrv')))
            suite.addTests(load_testsuite(loader, join(this_dir, 'cudapy')))
        else:
            print("skipped CUDA tests because GPU CC < 2.0")
    else:
        print("skipped CUDA tests")
    return suite
예제 #21
0
def load_tests(loader, tests, pattern):
    suite = unittest.TestSuite()
    this_dir = dirname(__file__)
    ensure_supported_ccs_initialized()
    suite.addTests(load_testsuite(loader, join(this_dir, 'nocuda')))
    if cuda.is_available():
        suite.addTests(load_testsuite(loader, join(this_dir, 'cudasim')))
        gpus = cuda.list_devices()
        if gpus and gpus[0].compute_capability >= (2, 0):
            suite.addTests(load_testsuite(loader, join(this_dir, 'cudadrv')))
            suite.addTests(load_testsuite(loader, join(this_dir, 'cudapy')))
        else:
            print("skipped CUDA tests because GPU CC < 2.0")
    else:
        print("skipped CUDA tests")
    return suite
예제 #22
0
def load_tests(loader, tests, pattern):
    suite = unittest.TestSuite()
    this_dir = dirname(__file__)
    suite.addTests(load_testsuite(loader, join(this_dir, 'nocuda')))
    if cuda.is_available():
        # Ensure that cudart.so is loaded and the list of supported compute
        # capabilities in the nvvm module is populated before a fork. This is
        # needed because some compilation tests don't require a CUDA context,
        # but do use NVVM to compile, and it is required that libcudart.so
        # should be loaded before a fork (note that the requirement is not
        # explicitly documented).
        cuda.cudadrv.nvvm.get_supported_ccs()
        suite.addTests(load_testsuite(loader, join(this_dir, 'cudasim')))
        gpus = cuda.list_devices()
        if gpus and gpus[0].compute_capability >= (2, 0):
            suite.addTests(load_testsuite(loader, join(this_dir, 'cudadrv')))
            suite.addTests(load_testsuite(loader, join(this_dir, 'cudapy')))
        else:
            print("skipped CUDA tests because GPU CC < 2.0")
    else:
        print("skipped CUDA tests")
    return suite
예제 #23
0
파일: __init__.py 프로젝트: hajs/numba
    def _do_discovery(self, argv, Loader=None):
        """The discovery process is complicated by the fact that:

        * different test suites live under different directories
        * some test suites may not be available (CUDA)
        * some tests may have to be run serially, even in the presence of the '-m' flag."""

        from numba import cuda

        join = os.path.join
        loader = unittest.TestLoader() if Loader is None else Loader()
        topdir = os.path.abspath(join(os.path.dirname(__file__), "../.."))
        base_tests = loader.discover(join(topdir, "numba/tests"), "test*.py", topdir)
        cuda_tests = [loader.discover(join(topdir, "numba/cuda/tests/nocuda"), "test*.py", topdir)]
        if cuda.is_available():
            gpus = cuda.list_devices()
            if gpus and gpus[0].compute_capability >= (2, 0):
                cuda_tests.append(loader.discover(join(topdir, "numba/cuda/tests/cudadrv"), "test*.py", topdir))
                cuda_tests.append(loader.discover(join(topdir, "numba/cuda/tests/cudapy"), "test*.py", topdir))
            else:
                print("skipped CUDA tests because GPU CC < 2.0")
        else:
            print("skipped CUDA tests")
        self.test = suite.TestSuite(tests=(base_tests, SerialSuite(cuda_tests)))
예제 #24
0
파일: numba_entry.py 프로젝트: yuguen/numba
def get_sys_info():
    # delay these imports until now as they are only needed in this
    # function which then exits.
    import platform
    import json
    from numba import cuda as cu
    from numba.cuda import cudadrv
    from numba.cuda.cudadrv.driver import driver as cudriver
    import textwrap as tw
    import ctypes as ct
    import llvmlite.binding as llvmbind
    import locale
    from datetime import datetime
    from itertools import chain
    from subprocess import check_output, CalledProcessError

    try:
        fmt = "%-21s : %-s"
        print("-" * 80)
        print("__Time Stamp__")
        print(datetime.utcnow())
        print("")

        print("__Hardware Information__")
        print(fmt % ("Machine", platform.machine()))
        print(fmt % ("CPU Name", llvmbind.get_host_cpu_name()))
        try:
            featuremap = llvmbind.get_host_cpu_features()
        except RuntimeError:
            print(fmt % ("CPU Features", "NA"))
        else:
            features = sorted([key for key, value in featuremap.items()
                               if value])
            cpu_feat = tw.fill(' '.join(features), 80)
            print(fmt % ("CPU Features", ""))
            print(cpu_feat)
        print("")

        print("__OS Information__")
        print(fmt % ("Platform", platform.platform(aliased=True)))
        print(fmt % ("Release", platform.release()))
        system_name = platform.system()
        print(fmt % ("System Name", system_name))
        print(fmt % ("Version", platform.version()))
        try:
            if system_name == 'Linux':
                info = platform.linux_distribution()
            elif system_name == 'Windows':
                info = platform.win32_ver()
            elif system_name == 'Darwin':
                info = platform.mac_ver()
            else:
                raise RuntimeError("Unknown system.")
            buf = ''.join([x
                           if x != '' else ' '
                           for x in list(chain.from_iterable(info))])
            print(fmt % ("OS specific info", buf))

            if system_name == 'Linux':
                print(fmt % ("glibc info", ' '.join(platform.libc_ver())))
        except:
            print("Error: System name incorrectly identified or unknown.")
        print("")

        print("__Python Information__")
        print(fmt % ("Python Compiler", platform.python_compiler()))
        print(
            fmt %
            ("Python Implementation",
             platform.python_implementation()))
        print(fmt % ("Python Version", platform.python_version()))
        print(
            fmt %
            ("Python Locale ", ' '.join(
                [x for x in locale.getdefaultlocale() if x is not None])))

        print("")
        print("__LLVM information__")
        print(
            fmt %
            ("LLVM version", '.'.join(
                [str(k) for k in llvmbind.llvm_version_info])))

        print("")
        print("__CUDA Information__")
        # Look for GPUs
        try:
            cu.list_devices()[0]  # will a device initialise?
        except BaseException as e:
            msg_not_found = "CUDA driver library cannot be found"
            msg_disabled_by_user = "******"
            msg_end = " or no CUDA enabled devices are present."
            msg_generic_problem = "Error: CUDA device intialisation problem."
            msg = getattr(e, 'msg', None)
            if msg is not None:
                if msg_not_found in msg:
                    err_msg = msg_not_found + msg_end
                elif msg_disabled_by_user in msg:
                    err_msg = msg_disabled_by_user + msg_end
                else:
                    err_msg = msg_generic_problem + " Message:" + msg
            else:
                err_msg = msg_generic_problem + " " + str(e)
            # Best effort error report
            print("%s\nError class: %s" % (err_msg, str(type(e))))
        else:
            try:
                cu.detect()
                dv = ct.c_int(0)
                cudriver.cuDriverGetVersion(ct.byref(dv))
                print(fmt % ("CUDA driver version", dv.value))
                print("CUDA libraries:")
                cudadrv.libs.test(sys.platform, print_paths=False)
            except:
                print(
                    "Error: Probing CUDA failed (device and driver present, runtime problem?)\n")

        # Look for conda and conda information
        print("")
        print("__Conda Information__")
        cmd = ["conda", "info", "--json"]
        try:
            conda_out = check_output(cmd)
        except Exception as e:
            print(
                "Conda not present/not working.\nError was %s\n" % e)
        else:
            data = ''.join(conda_out.decode("utf-8").splitlines())
            jsond = json.loads(data)
            keys = ['conda_build_version',
                    'conda_env_version',
                    'platform',
                    'python_version',
                    'root_writable']
            for k in keys:
                try:
                    print(fmt % (k, jsond[k]))
                except KeyError:
                    pass

            # get info about current environment
            cmd = ["conda", "list"]
            try:
                conda_out = check_output(cmd)
            except CalledProcessError as e:
                print("Error: Conda command failed. Error was %s\n" % e.output)
            else:
                print("")
                print("__Current Conda Env__")
                data = conda_out.decode("utf-8").splitlines()
                for k in data:
                    if k[0] != '#':  # don't show where the env is, personal data
                        print(k)

        print("-" * 80)

    except Exception as e:
        print("Error: The system reporting tool has failed unexpectedly.")
        print("Exception was:")
        print(e)

    finally:
        print(
            "%s" %
            "If requested, please copy and paste the information between\n"
            "the dashed (----) lines, or from a given specific section as\n"
            "appropriate.\n\n"
            "=============================================================\n"
            "IMPORTANT: Please ensure that you are happy with sharing the\n"
            "contents of the information present, any information that you\n"
            "wish to keep private you should remove before sharing.\n"
            "=============================================================\n")
예제 #25
0
from __future__ import print_function
import numpy as np
from math import ceil
import threading
from numba import cuda

print('System has %d CUDA devices' % len(cuda.list_devices()))

signature = 'void(int32[:], int32[:])'


def kernel(dst, src):
    '''A simple kernel that adds 1 to every item
    '''
    i = cuda.grid(1)
    if i >= dst.shape[0]:
        return
    dst[i] = src[i] + 1


# Numba compiler is not threadsafe
compiler_lock = threading.Lock()


def device_controller(cid):
    cuda.select_device(cid)  # bind device to thread
    device = cuda.get_current_device()  # get current device

    # print some information about the CUDA card
    prefix = '[%s]' % device
    print(prefix, 'device_controller', cid, '| CC', device.COMPUTE_CAPABILITY)
예제 #26
0
# distribute sub boxes among processors
inboxes = heffte.split_world(world, proc_i)
outboxes = heffte.split_world(world, proc_o)

# create plan
fft = heffte.fft3d(heffte.backend.fftw, inboxes[me], outboxes[me], mpi_comm)

# Initialize data
device = 'nvdia_gpu'
# device = 'amd_gpu'
make_data(fftsize, device)

# ------------------------------
print("NVDIA GPUs available = ")
print(cuda.list_devices())
# ------------------------------

mpi_comm.Barrier()
cuda.synchronize()
# roc.barrier(roc.CLK_GLOBAL_MEM_FENCE)

time1 = MPI.Wtime()

fft.forward(work, work2, heffte.scale.none)

cuda.synchronize()
# roc.barrier(roc.CLK_GLOBAL_MEM_FENCE)
mpi_comm.Barrier()

print("---------------------------")
예제 #27
0
파일: numba_entry.py 프로젝트: numba/numba
def get_sys_info():
    # delay these imports until now as they are only needed in this
    # function which then exits.
    import platform
    import json
    import multiprocessing
    from numba import config
    from numba import cuda as cu
    from numba.cuda import cudadrv
    from numba.cuda.cudadrv.driver import driver as cudriver
    from numba import roc
    from numba.roc.hlc import hlc, libhlc
    import textwrap as tw
    import ctypes as ct
    import llvmlite.binding as llvmbind
    import locale
    from datetime import datetime
    from itertools import chain
    from subprocess import check_output, CalledProcessError

    try:
        fmt = "%-45s : %-s"
        print("-" * 80)
        print("__Time Stamp__")
        print(datetime.utcnow())
        print("")

        print("__Hardware Information__")
        system_name = platform.system()
        print(fmt % ("Machine", platform.machine()))
        print(fmt % ("CPU Name", llvmbind.get_host_cpu_name()))
        if system_name == 'Linux':
            strmatch = 'Cpus_allowed'
            try:
                loc = '/proc/self/status'
                with open(loc, 'rt') as f:
                    proc_stat = f.read().splitlines()
                    for x in proc_stat:
                        if x.startswith(strmatch):
                            if x.startswith('%s:' % strmatch):
                                hexnum = '0x%s' % x.split(':')[1].strip()
                                acc_cpus = int(hexnum, 16)
                                _n = str(bin(acc_cpus).count('1'))
                                print(fmt % ("Number of accessible CPU cores",
                                                _n))
                            elif x.startswith('%s_list:' % strmatch):
                                _a = x.split(':')[1].strip()
                                print(fmt % ("Listed accessible CPUs cores",
                                                _a))
            except BaseException:
                print(fmt % ("CPU count", multiprocessing.cpu_count()))
            # See if CFS is in place
            # https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
            try:
                def scrape_lines(loc):
                    with open(loc, 'rt') as f:
                        return f.read().splitlines()
                loc = '/sys/fs/cgroup/cpuacct/cpu.cfs_period_us'
                cfs_period = int(scrape_lines(loc)[0])
                loc = '/sys/fs/cgroup/cpuacct/cpu.cfs_quota_us'
                cfs_quota = int(scrape_lines(loc)[0])
                if cfs_quota == -1:
                    print(fmt % ("CFS restrictions", "None"))
                else:
                    runtime_amount = float(cfs_quota)/float(cfs_period)
                    print(fmt % ("CFS restrictions (CPUs worth of runtime)",
                                 runtime_amount))
            except BaseException:
                print(fmt % ("CFS restrictions", 'Information not available'))
        else:
            print(fmt % ("CPU count", multiprocessing.cpu_count()))

        try:
            featuremap = llvmbind.get_host_cpu_features()
        except RuntimeError:
            print(fmt % ("CPU Features", "NA"))
        else:
            features = sorted([key for key, value in featuremap.items()
                               if value])
            cpu_feat = tw.fill(' '.join(features), 80)
            print(fmt % ("CPU Features", ""))
            print(cpu_feat)
        print("")

        print("__OS Information__")
        print(fmt % ("Platform", platform.platform(aliased=True)))
        print(fmt % ("Release", platform.release()))
        print(fmt % ("System Name", system_name))
        print(fmt % ("Version", platform.version()))
        try:
            if system_name == 'Linux':
                info = platform.linux_distribution()
            elif system_name == 'Windows':
                info = platform.win32_ver()
            elif system_name == 'Darwin':
                info = platform.mac_ver()
            else:
                raise RuntimeError("Unknown system.")
            buf = ''.join([x
                           if x != '' else ' '
                           for x in list(chain.from_iterable(info))])
            print(fmt % ("OS specific info", buf))

            if system_name == 'Linux':
                print(fmt % ("glibc info", ' '.join(platform.libc_ver())))
        except:
            print("Error: System name incorrectly identified or unknown.")
        print("")

        print("__Python Information__")
        print(fmt % ("Python Compiler", platform.python_compiler()))
        print(
            fmt %
            ("Python Implementation",
             platform.python_implementation()))
        print(fmt % ("Python Version", platform.python_version()))
        lcl = []
        try:
            for x in locale.getdefaultlocale():
                if x is not None:
                    lcl.append(x)
        except BaseException as e:
            lcl.append(str(e))
        print(fmt % ("Python Locale ", ' '.join(lcl)))

        print("")
        print("__LLVM information__")
        print(
            fmt %
            ("LLVM version", '.'.join(
                [str(k) for k in llvmbind.llvm_version_info])))

        print("")
        print("__CUDA Information__")
        # Look for GPUs
        try:
            cu.list_devices()[0]  # will a device initialise?
        except BaseException as e:
            msg_not_found = "CUDA driver library cannot be found"
            msg_disabled_by_user = "******"
            msg_end = " or no CUDA enabled devices are present."
            msg_generic_problem = "Error: CUDA device intialisation problem."
            msg = getattr(e, 'msg', None)
            if msg is not None:
                if msg_not_found in msg:
                    err_msg = msg_not_found + msg_end
                elif msg_disabled_by_user in msg:
                    err_msg = msg_disabled_by_user + msg_end
                else:
                    err_msg = msg_generic_problem + " Message:" + msg
            else:
                err_msg = msg_generic_problem + " " + str(e)
            # Best effort error report
            print("%s\nError class: %s" % (err_msg, str(type(e))))
        else:
            try:
                cu.detect()
                dv = ct.c_int(0)
                cudriver.cuDriverGetVersion(ct.byref(dv))
                print(fmt % ("CUDA driver version", dv.value))
                print("CUDA libraries:")
                cudadrv.libs.test(sys.platform, print_paths=False)
            except:
                print(
                    "Error: Probing CUDA failed (device and driver present, runtime problem?)\n")

        print("")
        print("__ROC Information__")
        roc_is_available = roc.is_available()
        print(fmt % ("ROC available", roc_is_available))

        toolchains = []
        try:
            libhlc.HLC()
            toolchains.append('librocmlite library')
        except:
            pass
        try:
            cmd = hlc.CmdLine().check_tooling()
            toolchains.append('ROC command line tools')
        except:
            pass

        # if no ROC try and report why
        if not roc_is_available:
            from numba.roc.hsadrv.driver import hsa
            try:
                hsa.is_available
            except BaseException as e:
                msg = str(e)
            else:
               msg = 'No ROC toolchains found.'
            print(fmt % ("Error initialising ROC due to", msg))

        if toolchains:
            print(fmt % ("Available Toolchains", ', '.join(toolchains)))

        try:
            # ROC might not be available due to lack of tool chain, but HSA
            # agents may be listed
            from numba.roc.hsadrv.driver import hsa, dgpu_count
            decode = lambda x: x.decode('utf-8') if isinstance(x, bytes) else x
            print("\nFound %s HSA Agents:" % len(hsa.agents))
            for i, agent in enumerate(hsa.agents):
                print('Agent id  : %s' % i)
                print('    vendor: %s' % decode(agent.vendor_name))
                print('      name: %s' % decode(agent.name))
                print('      type: %s' % agent.device)
                print("")

            _dgpus = []
            for a in hsa.agents:
                if a.is_component and a.device == 'GPU':
                   _dgpus.append(decode(a.name))
            print(fmt % ("Found %s discrete GPU(s)" % dgpu_count(), \
                  ', '.join(_dgpus)))
        except Exception as e:
            print("No HSA Agents found, encountered exception when searching:")
            print(e)


        print("")
        print("__SVML Information__")
        # replicate some SVML detection logic from numba.__init__ here.
        # if SVML load fails in numba.__init__ the splitting of the logic
        # here will help diagnosis of the underlying issue
        have_svml_library = True
        try:
            if sys.platform.startswith('linux'):
                llvmbind.load_library_permanently("libsvml.so")
            elif sys.platform.startswith('darwin'):
                llvmbind.load_library_permanently("libsvml.dylib")
            elif sys.platform.startswith('win'):
                llvmbind.load_library_permanently("svml_dispmd")
            else:
                have_svml_library = False
        except:
            have_svml_library = False
        func = getattr(llvmbind.targets, "has_svml", None)
        llvm_svml_patched = func() if func is not None else False
        svml_operational = (config.USING_SVML and llvm_svml_patched \
                            and have_svml_library)
        print(fmt % ("SVML state, config.USING_SVML", config.USING_SVML))
        print(fmt % ("SVML library found and loaded", have_svml_library))
        print(fmt % ("llvmlite using SVML patched LLVM", llvm_svml_patched))
        print(fmt % ("SVML operational", svml_operational))

        # Check which threading backends are available.
        print("")
        print("__Threading Layer Information__")
        def parse_error(e, backend):
            # parses a linux based error message, this is to provide feedback
            # and hide user paths etc
            try:
                path, problem, symbol =  [x.strip() for x in e.msg.split(':')]
                extn_dso = os.path.split(path)[1]
                if backend in extn_dso:
                    return "%s: %s" % (problem, symbol)
            except BaseException:
                pass
            return "Unknown import problem."

        try:
            from numba.npyufunc import tbbpool
            print(fmt % ("TBB Threading layer available", True))
        except ImportError as e:
            # might be a missing symbol due to e.g. tbb libraries missing
            print(fmt % ("TBB Threading layer available", False))
            print(fmt % ("+--> Disabled due to",
                         parse_error(e, 'tbbpool')))

        try:
            from numba.npyufunc import omppool
            print(fmt % ("OpenMP Threading layer available", True))
        except ImportError as e:
            print(fmt % ("OpenMP Threading layer available", False))
            print(fmt % ("+--> Disabled due to",
                         parse_error(e, 'omppool')))

        try:
            from numba.npyufunc import workqueue
            print(fmt % ("Workqueue Threading layer available", True))
        except ImportError as e:
            print(fmt % ("Workqueue Threading layer available", False))
            print(fmt % ("+--> Disabled due to",
                         parse_error(e, 'workqueue')))

        # look for numba env vars that are set
        print("")
        print("__Numba Environment Variable Information__")
        _envvar_found = False
        for k, v in os.environ.items():
            if k.startswith('NUMBA_'):
                print(fmt % (k, v))
                _envvar_found = True
        if not _envvar_found:
            print("None set.")

        # Look for conda and conda information
        print("")
        print("__Conda Information__")
        cmd = ["conda", "info", "--json"]
        try:
            conda_out = check_output(cmd)
        except Exception as e:
            print(
                "Conda not present/not working.\nError was %s\n" % e)
        else:
            data = ''.join(conda_out.decode("utf-8").splitlines())
            jsond = json.loads(data)
            keys = ['conda_build_version',
                    'conda_env_version',
                    'platform',
                    'python_version',
                    'root_writable']
            for k in keys:
                try:
                    print(fmt % (k, jsond[k]))
                except KeyError:
                    pass

            # get info about current environment
            cmd = ["conda", "list"]
            try:
                conda_out = check_output(cmd)
            except CalledProcessError as e:
                print("Error: Conda command failed. Error was %s\n" % e.output)
            else:
                print("")
                print("__Current Conda Env__")
                data = conda_out.decode("utf-8").splitlines()
                for k in data:
                    if k[0] != '#':  # don't show where the env is, personal data
                        print(k)

        print("-" * 80)

    except Exception as e:
        print("Error: The system reporting tool has failed unexpectedly.")
        print("Exception was:")
        print(e)

    finally:
        print(
            "%s" %
            "If requested, please copy and paste the information between\n"
            "the dashed (----) lines, or from a given specific section as\n"
            "appropriate.\n\n"
            "=============================================================\n"
            "IMPORTANT: Please ensure that you are happy with sharing the\n"
            "contents of the information present, any information that you\n"
            "wish to keep private you should remove before sharing.\n"
            "=============================================================\n")
예제 #28
0
def get_sys_info():
    # delay these imports until now as they are only needed in this
    # function which then exits.
    import platform
    import json
    from numba import cuda as cu
    from numba.cuda import cudadrv
    from numba.cuda.cudadrv.driver import driver as cudriver
    import textwrap as tw
    import ctypes as ct
    import llvmlite.binding as llvmbind
    import locale
    from datetime import datetime
    from itertools import chain
    from subprocess import check_output, CalledProcessError

    try:
        fmt = "%-21s : %-s"
        print("-" * 80)
        print("__Time Stamp__")
        print(datetime.utcnow())
        print("")

        print("__Hardware Information__")
        print(fmt % ("Machine", platform.machine()))
        print(fmt % ("CPU Name", llvmbind.get_host_cpu_name()))
        try:
            featuremap = llvmbind.get_host_cpu_features()
        except RuntimeError:
            print(fmt % ("CPU Features", "NA"))
        else:
            features = sorted(
                [key for key, value in featuremap.items() if value])
            cpu_feat = tw.fill(' '.join(features), 80)
            print(fmt % ("CPU Features", ""))
            print(cpu_feat)
        print("")

        print("__OS Information__")
        print(fmt % ("Platform", platform.platform(aliased=True)))
        print(fmt % ("Release", platform.release()))
        system_name = platform.system()
        print(fmt % ("System Name", system_name))
        print(fmt % ("Version", platform.version()))
        try:
            if system_name == 'Linux':
                info = platform.linux_distribution()
            elif system_name == 'Windows':
                info = platform.win32_ver()
            elif system_name == 'Darwin':
                info = platform.mac_ver()
            else:
                raise RuntimeError("Unknown system.")
            buf = ''.join([
                x if x != '' else ' ' for x in list(chain.from_iterable(info))
            ])
            print(fmt % ("OS specific info", buf))

            if system_name == 'Linux':
                print(fmt % ("glibc info", ' '.join(platform.libc_ver())))
        except:
            print("Error: System name incorrectly identified or unknown.")
        print("")

        print("__Python Information__")
        print(fmt % ("Python Compiler", platform.python_compiler()))
        print(fmt %
              ("Python Implementation", platform.python_implementation()))
        print(fmt % ("Python Version", platform.python_version()))
        print(fmt % ("Python Locale ", ' '.join(
            [x for x in locale.getdefaultlocale() if x is not None])))

        print("")
        print("__LLVM information__")
        print(fmt % ("LLVM version", '.'.join(
            [str(k) for k in llvmbind.llvm_version_info])))

        print("")
        print("__CUDA Information__")
        # Look for GPUs
        try:
            cu.list_devices()[0]  # will a device initialise?
        except BaseException as e:
            msg_not_found = "CUDA driver library cannot be found"
            msg_disabled_by_user = "******"
            msg_end = " or no CUDA enabled devices are present."
            msg_generic_problem = "Error: CUDA device intialisation problem."
            msg = getattr(e, 'msg', None)
            if msg is not None:
                if msg_not_found in msg:
                    err_msg = msg_not_found + msg_end
                elif msg_disabled_by_user in msg:
                    err_msg = msg_disabled_by_user + msg_end
                else:
                    err_msg = msg_generic_problem + " Message:" + msg
            else:
                err_msg = msg_generic_problem + " " + str(e)
            # Best effort error report
            print("%s\nError class: %s" % (err_msg, str(type(e))))
        else:
            try:
                cu.detect()
                dv = ct.c_int(0)
                cudriver.cuDriverGetVersion(ct.byref(dv))
                print(fmt % ("CUDA driver version", dv.value))
                print("CUDA libraries:")
                cudadrv.libs.test(sys.platform, print_paths=False)
            except:
                print(
                    "Error: Probing CUDA failed (device and driver present, runtime problem?)\n"
                )

        # Look for conda and conda information
        print("")
        print("__Conda Information__")
        cmd = ["conda", "info", "--json"]
        try:
            conda_out = check_output(cmd)
        except Exception as e:
            print("Conda not present/not working.\nError was %s\n" % e)
        else:
            data = ''.join(conda_out.decode("utf-8").splitlines())
            jsond = json.loads(data)
            keys = [
                'conda_build_version', 'conda_env_version', 'platform',
                'python_version', 'root_writable'
            ]
            for k in keys:
                try:
                    print(fmt % (k, jsond[k]))
                except KeyError:
                    pass

            # get info about current environment
            cmd = ["conda", "list"]
            try:
                conda_out = check_output(cmd)
            except CalledProcessError as e:
                print("Error: Conda command failed. Error was %s\n" % e.output)
            else:
                print("")
                print("__Current Conda Env__")
                data = conda_out.decode("utf-8").splitlines()
                for k in data:
                    if k[0] != '#':  # don't show where the env is, personal data
                        print(k)

        print("-" * 80)

    except Exception as e:
        print("Error: The system reporting tool has failed unexpectedly.")
        print("Exception was:")
        print(e)

    finally:
        print(
            "%s" %
            "If requested, please copy and paste the information between\n"
            "the dashed (----) lines, or from a given specific section as\n"
            "appropriate.\n\n"
            "=============================================================\n"
            "IMPORTANT: Please ensure that you are happy with sharing the\n"
            "contents of the information present, any information that you\n"
            "wish to keep private you should remove before sharing.\n"
            "=============================================================\n")
예제 #29
0
def get_sys_info():
    # delay these imports until now as they are only needed in this
    # function which then exits.
    import platform
    import json
    from numba import config
    from numba import cuda as cu
    from numba.cuda import cudadrv
    from numba.cuda.cudadrv.driver import driver as cudriver
    from numba import roc
    from numba.roc.hlc import hlc, libhlc
    import textwrap as tw
    import ctypes as ct
    import llvmlite.binding as llvmbind
    import locale
    from datetime import datetime
    from itertools import chain
    from subprocess import check_output, CalledProcessError

    try:
        fmt = "%-35s : %-s"
        print("-" * 80)
        print("__Time Stamp__")
        print(datetime.utcnow())
        print("")

        print("__Hardware Information__")
        print(fmt % ("Machine", platform.machine()))
        print(fmt % ("CPU Name", llvmbind.get_host_cpu_name()))
        try:
            featuremap = llvmbind.get_host_cpu_features()
        except RuntimeError:
            print(fmt % ("CPU Features", "NA"))
        else:
            features = sorted(
                [key for key, value in featuremap.items() if value])
            cpu_feat = tw.fill(' '.join(features), 80)
            print(fmt % ("CPU Features", ""))
            print(cpu_feat)
        print("")

        print("__OS Information__")
        print(fmt % ("Platform", platform.platform(aliased=True)))
        print(fmt % ("Release", platform.release()))
        system_name = platform.system()
        print(fmt % ("System Name", system_name))
        print(fmt % ("Version", platform.version()))
        try:
            if system_name == 'Linux':
                info = platform.linux_distribution()
            elif system_name == 'Windows':
                info = platform.win32_ver()
            elif system_name == 'Darwin':
                info = platform.mac_ver()
            else:
                raise RuntimeError("Unknown system.")
            buf = ''.join([
                x if x != '' else ' ' for x in list(chain.from_iterable(info))
            ])
            print(fmt % ("OS specific info", buf))

            if system_name == 'Linux':
                print(fmt % ("glibc info", ' '.join(platform.libc_ver())))
        except:
            print("Error: System name incorrectly identified or unknown.")
        print("")

        print("__Python Information__")
        print(fmt % ("Python Compiler", platform.python_compiler()))
        print(fmt %
              ("Python Implementation", platform.python_implementation()))
        print(fmt % ("Python Version", platform.python_version()))
        print(fmt % ("Python Locale ", ' '.join(
            [x for x in locale.getdefaultlocale() if x is not None])))

        print("")
        print("__LLVM information__")
        print(fmt % ("LLVM version", '.'.join(
            [str(k) for k in llvmbind.llvm_version_info])))

        print("")
        print("__CUDA Information__")
        # Look for GPUs
        try:
            cu.list_devices()[0]  # will a device initialise?
        except BaseException as e:
            msg_not_found = "CUDA driver library cannot be found"
            msg_disabled_by_user = "******"
            msg_end = " or no CUDA enabled devices are present."
            msg_generic_problem = "Error: CUDA device intialisation problem."
            msg = getattr(e, 'msg', None)
            if msg is not None:
                if msg_not_found in msg:
                    err_msg = msg_not_found + msg_end
                elif msg_disabled_by_user in msg:
                    err_msg = msg_disabled_by_user + msg_end
                else:
                    err_msg = msg_generic_problem + " Message:" + msg
            else:
                err_msg = msg_generic_problem + " " + str(e)
            # Best effort error report
            print("%s\nError class: %s" % (err_msg, str(type(e))))
        else:
            try:
                cu.detect()
                dv = ct.c_int(0)
                cudriver.cuDriverGetVersion(ct.byref(dv))
                print(fmt % ("CUDA driver version", dv.value))
                print("CUDA libraries:")
                cudadrv.libs.test(sys.platform, print_paths=False)
            except:
                print(
                    "Error: Probing CUDA failed (device and driver present, runtime problem?)\n"
                )

        print("")
        print("__ROC Information__")
        roc_is_available = roc.is_available()
        print(fmt % ("ROC available", roc_is_available))

        toolchains = []
        try:
            libhlc.HLC()
            toolchains.append('librocmlite library')
        except:
            pass
        try:
            cmd = hlc.CmdLine().check_tooling()
            toolchains.append('ROC command line tools')
        except:
            pass

        # if no ROC try and report why
        if not roc_is_available:
            from numba.roc.hsadrv.driver import hsa
            try:
                hsa.is_available
            except BaseException as e:
                msg = str(e)
            else:
                msg = 'No ROC toolchains found.'
            print(fmt % ("Error initialising ROC due to", msg))

        if toolchains:
            print(fmt % ("Available Toolchains", ', '.join(toolchains)))

        try:
            # ROC might not be available due to lack of tool chain, but HSA
            # agents may be listed
            from numba.roc.hsadrv.driver import hsa, dgpu_count
            print("\nFound %s HSA Agents:" % len(hsa.agents))
            for i, agent in enumerate(hsa.agents):
                print('Agent id  : %s' % i)
                print('    vendor: %s' % agent.vendor_name)
                print('      name: %s' % agent.name)
                print('      type: %s' % agent.device)
                print("")

            _dgpus = []
            for a in hsa.agents:
                if a.is_component and a.device == 'GPU':
                    _dgpus.append(a.name)
            print(fmt % ("Found %s discrete GPU(s)" % dgpu_count(), \
                  ', '.join(_dgpus)))
        except Exception as e:
            print("No HSA Agents found, encountered exception when searching:")
            print(e)

        print("")
        print("__SVML Information__")
        # replicate some SVML detection logic from numba.__init__ here.
        # if SVML load fails in numba.__init__ the splitting of the logic
        # here will help diagnosis of the underlying issue
        have_svml_library = True
        try:
            if sys.platform.startswith('linux'):
                llvmbind.load_library_permanently("libsvml.so")
            elif sys.platform.startswith('darwin'):
                llvmbind.load_library_permanently("libsvml.dylib")
            elif sys.platform.startswith('win'):
                llvmbind.load_library_permanently("svml_dispmd")
            else:
                have_svml_library = False
        except:
            have_svml_library = False
        func = getattr(llvmbind.targets, "has_svml", None)
        llvm_svml_patched = func() if func is not None else False
        svml_operational = (config.USING_SVML and llvm_svml_patched \
                            and have_svml_library)
        print(fmt % ("SVML state, config.USING_SVML", config.USING_SVML))
        print(fmt % ("SVML library found and loaded", have_svml_library))
        print(fmt % ("llvmlite using SVML patched LLVM", llvm_svml_patched))
        print(fmt % ("SVML operational:", svml_operational))

        # Look for conda and conda information
        print("")
        print("__Conda Information__")
        cmd = ["conda", "info", "--json"]
        try:
            conda_out = check_output(cmd)
        except Exception as e:
            print("Conda not present/not working.\nError was %s\n" % e)
        else:
            data = ''.join(conda_out.decode("utf-8").splitlines())
            jsond = json.loads(data)
            keys = [
                'conda_build_version', 'conda_env_version', 'platform',
                'python_version', 'root_writable'
            ]
            for k in keys:
                try:
                    print(fmt % (k, jsond[k]))
                except KeyError:
                    pass

            # get info about current environment
            cmd = ["conda", "list"]
            try:
                conda_out = check_output(cmd)
            except CalledProcessError as e:
                print("Error: Conda command failed. Error was %s\n" % e.output)
            else:
                print("")
                print("__Current Conda Env__")
                data = conda_out.decode("utf-8").splitlines()
                for k in data:
                    if k[0] != '#':  # don't show where the env is, personal data
                        print(k)

        print("-" * 80)

    except Exception as e:
        print("Error: The system reporting tool has failed unexpectedly.")
        print("Exception was:")
        print(e)

    finally:
        print(
            "%s" %
            "If requested, please copy and paste the information between\n"
            "the dashed (----) lines, or from a given specific section as\n"
            "appropriate.\n\n"
            "=============================================================\n"
            "IMPORTANT: Please ensure that you are happy with sharing the\n"
            "contents of the information present, any information that you\n"
            "wish to keep private you should remove before sharing.\n"
            "=============================================================\n")
예제 #30
0
def get_sys_info():
    # delay these imports until now as they are only needed in this
    # function which then exits.
    import platform
    import json
    from numba import config
    from numba import cuda as cu
    from numba.cuda import cudadrv
    from numba.cuda.cudadrv.driver import driver as cudriver
    from numba import roc
    from numba.roc.hlc import hlc, libhlc
    import textwrap as tw
    import ctypes as ct
    import llvmlite.binding as llvmbind
    import locale
    from datetime import datetime
    from itertools import chain
    from subprocess import check_output, CalledProcessError

    try:
        fmt = "%-35s : %-s"
        print("-" * 80)
        print("__Time Stamp__")
        print(datetime.utcnow())
        print("")

        print("__Hardware Information__")
        print(fmt % ("Machine", platform.machine()))
        print(fmt % ("CPU Name", llvmbind.get_host_cpu_name()))
        try:
            featuremap = llvmbind.get_host_cpu_features()
        except RuntimeError:
            print(fmt % ("CPU Features", "NA"))
        else:
            features = sorted([key for key, value in featuremap.items()
                               if value])
            cpu_feat = tw.fill(' '.join(features), 80)
            print(fmt % ("CPU Features", ""))
            print(cpu_feat)
        print("")

        print("__OS Information__")
        print(fmt % ("Platform", platform.platform(aliased=True)))
        print(fmt % ("Release", platform.release()))
        system_name = platform.system()
        print(fmt % ("System Name", system_name))
        print(fmt % ("Version", platform.version()))
        try:
            if system_name == 'Linux':
                info = platform.linux_distribution()
            elif system_name == 'Windows':
                info = platform.win32_ver()
            elif system_name == 'Darwin':
                info = platform.mac_ver()
            else:
                raise RuntimeError("Unknown system.")
            buf = ''.join([x
                           if x != '' else ' '
                           for x in list(chain.from_iterable(info))])
            print(fmt % ("OS specific info", buf))

            if system_name == 'Linux':
                print(fmt % ("glibc info", ' '.join(platform.libc_ver())))
        except:
            print("Error: System name incorrectly identified or unknown.")
        print("")

        print("__Python Information__")
        print(fmt % ("Python Compiler", platform.python_compiler()))
        print(
            fmt %
            ("Python Implementation",
             platform.python_implementation()))
        print(fmt % ("Python Version", platform.python_version()))
        print(
            fmt %
            ("Python Locale ", ' '.join(
                [x for x in locale.getdefaultlocale() if x is not None])))

        print("")
        print("__LLVM information__")
        print(
            fmt %
            ("LLVM version", '.'.join(
                [str(k) for k in llvmbind.llvm_version_info])))

        print("")
        print("__CUDA Information__")
        # Look for GPUs
        try:
            cu.list_devices()[0]  # will a device initialise?
        except BaseException as e:
            msg_not_found = "CUDA driver library cannot be found"
            msg_disabled_by_user = "******"
            msg_end = " or no CUDA enabled devices are present."
            msg_generic_problem = "Error: CUDA device intialisation problem."
            msg = getattr(e, 'msg', None)
            if msg is not None:
                if msg_not_found in msg:
                    err_msg = msg_not_found + msg_end
                elif msg_disabled_by_user in msg:
                    err_msg = msg_disabled_by_user + msg_end
                else:
                    err_msg = msg_generic_problem + " Message:" + msg
            else:
                err_msg = msg_generic_problem + " " + str(e)
            # Best effort error report
            print("%s\nError class: %s" % (err_msg, str(type(e))))
        else:
            try:
                cu.detect()
                dv = ct.c_int(0)
                cudriver.cuDriverGetVersion(ct.byref(dv))
                print(fmt % ("CUDA driver version", dv.value))
                print("CUDA libraries:")
                cudadrv.libs.test(sys.platform, print_paths=False)
            except:
                print(
                    "Error: Probing CUDA failed (device and driver present, runtime problem?)\n")

        print("")
        print("__ROC Information__")
        roc_is_available = roc.is_available()
        print(fmt % ("ROC available", roc_is_available))

        toolchains = []
        try:
            libhlc.HLC()
            toolchains.append('librocmlite library')
        except:
            pass
        try:
            cmd = hlc.CmdLine().check_tooling()
            toolchains.append('ROC command line tools')
        except:
            pass

        # if no ROC try and report why
        if not roc_is_available:
            from numba.roc.hsadrv.driver import hsa
            try:
                hsa.is_available
            except BaseException as e:
                msg = str(e)
            else:
               msg = 'No ROC toolchains found.'
            print(fmt % ("Error initialising ROC due to", msg))

        if toolchains:
            print(fmt % ("Available Toolchains", ', '.join(toolchains)))

        try:
            # ROC might not be available due to lack of tool chain, but HSA
            # agents may be listed
            from numba.roc.hsadrv.driver import hsa, dgpu_count
            decode = lambda x: x.decode('utf-8') if isinstance(x, bytes) else x
            print("\nFound %s HSA Agents:" % len(hsa.agents))
            for i, agent in enumerate(hsa.agents):
                print('Agent id  : %s' % i)
                print('    vendor: %s' % decode(agent.vendor_name))
                print('      name: %s' % decode(agent.name))
                print('      type: %s' % agent.device)
                print("")

            _dgpus = []
            for a in hsa.agents:
                if a.is_component and a.device == 'GPU':
                   _dgpus.append(decode(a.name))
            print(fmt % ("Found %s discrete GPU(s)" % dgpu_count(), \
                  ', '.join(_dgpus)))
        except Exception as e:
            print("No HSA Agents found, encountered exception when searching:")
            print(e)


        print("")
        print("__SVML Information__")
        # replicate some SVML detection logic from numba.__init__ here.
        # if SVML load fails in numba.__init__ the splitting of the logic
        # here will help diagnosis of the underlying issue
        have_svml_library = True
        try:
            if sys.platform.startswith('linux'):
                llvmbind.load_library_permanently("libsvml.so")
            elif sys.platform.startswith('darwin'):
                llvmbind.load_library_permanently("libsvml.dylib")
            elif sys.platform.startswith('win'):
                llvmbind.load_library_permanently("svml_dispmd")
            else:
                have_svml_library = False
        except:
            have_svml_library = False
        func = getattr(llvmbind.targets, "has_svml", None)
        llvm_svml_patched = func() if func is not None else False
        svml_operational = (config.USING_SVML and llvm_svml_patched \
                            and have_svml_library)
        print(fmt % ("SVML state, config.USING_SVML", config.USING_SVML))
        print(fmt % ("SVML library found and loaded", have_svml_library))
        print(fmt % ("llvmlite using SVML patched LLVM", llvm_svml_patched))
        print(fmt % ("SVML operational:", svml_operational))

        # Look for conda and conda information
        print("")
        print("__Conda Information__")
        cmd = ["conda", "info", "--json"]
        try:
            conda_out = check_output(cmd)
        except Exception as e:
            print(
                "Conda not present/not working.\nError was %s\n" % e)
        else:
            data = ''.join(conda_out.decode("utf-8").splitlines())
            jsond = json.loads(data)
            keys = ['conda_build_version',
                    'conda_env_version',
                    'platform',
                    'python_version',
                    'root_writable']
            for k in keys:
                try:
                    print(fmt % (k, jsond[k]))
                except KeyError:
                    pass

            # get info about current environment
            cmd = ["conda", "list"]
            try:
                conda_out = check_output(cmd)
            except CalledProcessError as e:
                print("Error: Conda command failed. Error was %s\n" % e.output)
            else:
                print("")
                print("__Current Conda Env__")
                data = conda_out.decode("utf-8").splitlines()
                for k in data:
                    if k[0] != '#':  # don't show where the env is, personal data
                        print(k)

        print("-" * 80)

    except Exception as e:
        print("Error: The system reporting tool has failed unexpectedly.")
        print("Exception was:")
        print(e)

    finally:
        print(
            "%s" %
            "If requested, please copy and paste the information between\n"
            "the dashed (----) lines, or from a given specific section as\n"
            "appropriate.\n\n"
            "=============================================================\n"
            "IMPORTANT: Please ensure that you are happy with sharing the\n"
            "contents of the information present, any information that you\n"
            "wish to keep private you should remove before sharing.\n"
            "=============================================================\n")
예제 #31
0
def get_sysinfo():

    # Gather the information that shouldn't raise exceptions
    sys_info = {
        _start: datetime.now(),
        _start_utc: datetime.utcnow(),
        _machine: platform.machine(),
        _cpu_name: llvmbind.get_host_cpu_name(),
        _cpu_count: multiprocessing.cpu_count(),
        _platform_name: platform.platform(aliased=True),
        _platform_release: platform.release(),
        _os_name: platform.system(),
        _os_version: platform.version(),
        _python_comp: platform.python_compiler(),
        _python_impl: platform.python_implementation(),
        _python_version: platform.python_version(),
        _numba_env_vars:
        {k: v
         for (k, v) in os.environ.items() if k.startswith('NUMBA_')},
        _numba_version: version_number,
        _llvm_version: '.'.join(str(i) for i in llvmbind.llvm_version_info),
        _llvmlite_version: llvmlite_version,
        _psutil: _psutil_import,
    }

    # CPU features
    try:
        feature_map = llvmbind.get_host_cpu_features()
    except RuntimeError as e:
        _error_log.append(f'Error (CPU features): {e}')
    else:
        features = sorted([key for key, value in feature_map.items() if value])
        sys_info[_cpu_features] = ' '.join(features)

    # Python locale
    # On MacOSX, getdefaultlocale can raise. Check again if Py > 3.7.5
    try:
        # If $LANG is unset, getdefaultlocale() can return (None, None), make
        # sure we can encode this as strings by casting explicitly.
        sys_info[_python_locale] = '.'.join(
            [str(i) for i in locale.getdefaultlocale()])
    except Exception as e:
        _error_log.append(f'Error (locale): {e}')

    # CUDA information
    try:
        cu.list_devices()[0]  # will a device initialise?
    except Exception as e:
        sys_info[_cu_dev_init] = False
        msg_not_found = "CUDA driver library cannot be found"
        msg_disabled_by_user = "******"
        msg_end = " or no CUDA enabled devices are present."
        msg_generic_problem = "CUDA device initialisation problem."
        msg = getattr(e, 'msg', None)
        if msg is not None:
            if msg_not_found in msg:
                err_msg = msg_not_found + msg_end
            elif msg_disabled_by_user in msg:
                err_msg = msg_disabled_by_user + msg_end
            else:
                err_msg = msg_generic_problem + " Message:" + msg
        else:
            err_msg = msg_generic_problem + " " + str(e)
        # Best effort error report
        _warning_log.append("Warning (cuda): %s\nException class: %s" %
                            (err_msg, str(type(e))))
    else:
        try:
            sys_info[_cu_dev_init] = True

            output = StringIO()
            with redirect_stdout(output):
                cu.detect()
            sys_info[_cu_detect_out] = output.getvalue()
            output.close()

            sys_info[_cu_drv_ver] = '%s.%s' % cudriver.get_version()
            sys_info[_cu_rt_ver] = '%s.%s' % curuntime.get_version()

            output = StringIO()
            with redirect_stdout(output):
                cudadrv.libs.test(sys.platform, print_paths=False)
            sys_info[_cu_lib_test] = output.getvalue()
            output.close()

            try:
                from cuda import cuda  # noqa: F401
                nvidia_bindings_available = True
            except ImportError:
                nvidia_bindings_available = False
            sys_info[_cu_nvidia_bindings] = nvidia_bindings_available

            nv_binding_used = bool(cudadrv.driver.USE_NV_BINDING)
            sys_info[_cu_nvidia_bindings_used] = nv_binding_used
        except Exception as e:
            _warning_log.append(
                "Warning (cuda): Probing CUDA failed "
                "(device and driver present, runtime problem?)\n"
                f"(cuda) {type(e)}: {e}")

    # NumPy information
    sys_info[_numpy_version] = np.version.full_version
    try:
        # NOTE: These consts were added in NumPy 1.20
        from numpy.core._multiarray_umath import (
            __cpu_features__,
            __cpu_dispatch__,
            __cpu_baseline__,
        )
    except ImportError:
        sys_info[_numpy_AVX512_SKX_detected] = False
    else:
        feat_filtered = [k for k, v in __cpu_features__.items() if v]
        sys_info[_numpy_supported_simd_features] = feat_filtered
        sys_info[_numpy_supported_simd_dispatch] = __cpu_dispatch__
        sys_info[_numpy_supported_simd_baseline] = __cpu_baseline__
        sys_info[_numpy_AVX512_SKX_detected] = \
            __cpu_features__.get("AVX512_SKX", False)

    # SVML information
    # Replicate some SVML detection logic from numba.__init__ here.
    # If SVML load fails in numba.__init__ the splitting of the logic
    # here will help diagnosing the underlying issue.
    svml_lib_loaded = True
    try:
        if sys.platform.startswith('linux'):
            llvmbind.load_library_permanently("libsvml.so")
        elif sys.platform.startswith('darwin'):
            llvmbind.load_library_permanently("libsvml.dylib")
        elif sys.platform.startswith('win'):
            llvmbind.load_library_permanently("svml_dispmd")
        else:
            svml_lib_loaded = False
    except Exception:
        svml_lib_loaded = False
    func = getattr(llvmbind.targets, "has_svml", None)
    sys_info[_llvm_svml_patched] = func() if func else False
    sys_info[_svml_state] = config.USING_SVML
    sys_info[_svml_loaded] = svml_lib_loaded
    sys_info[_svml_operational] = all((
        sys_info[_svml_state],
        sys_info[_svml_loaded],
        sys_info[_llvm_svml_patched],
    ))

    # Check which threading backends are available.
    def parse_error(e, backend):
        # parses a linux based error message, this is to provide feedback
        # and hide user paths etc
        try:
            path, problem, symbol = [x.strip() for x in e.msg.split(':')]
            extn_dso = os.path.split(path)[1]
            if backend in extn_dso:
                return "%s: %s" % (problem, symbol)
        except Exception:
            pass
        return "Unknown import problem."

    try:
        # check import is ok, this means the DSO linkage is working
        from numba.np.ufunc import tbbpool  # NOQA
        # check that the version is compatible, this is a check performed at
        # runtime (well, compile time), it will also ImportError if there's
        # a problem.
        from numba.np.ufunc.parallel import _check_tbb_version_compatible
        _check_tbb_version_compatible()
        sys_info[_tbb_thread] = True
    except ImportError as e:
        # might be a missing symbol due to e.g. tbb libraries missing
        sys_info[_tbb_thread] = False
        sys_info[_tbb_error] = parse_error(e, 'tbbpool')

    try:
        from numba.np.ufunc import omppool
        sys_info[_openmp_thread] = True
        sys_info[_openmp_vendor] = omppool.openmp_vendor
    except ImportError as e:
        sys_info[_openmp_thread] = False
        sys_info[_openmp_error] = parse_error(e, 'omppool')

    try:
        from numba.np.ufunc import workqueue  # NOQA
        sys_info[_wkq_thread] = True
    except ImportError as e:
        sys_info[_wkq_thread] = True
        sys_info[_wkq_error] = parse_error(e, 'workqueue')

    # Look for conda and installed packages information
    cmd = ('conda', 'info', '--json')
    try:
        conda_out = check_output(cmd)
    except Exception as e:
        _warning_log.append(f'Warning: Conda not available.\n Error was {e}\n')
        # Conda is not available, try pip list to list installed packages
        cmd = (sys.executable, '-m', 'pip', 'list')
        try:
            reqs = check_output(cmd)
        except Exception as e:
            _error_log.append(f'Error (pip): {e}')
        else:
            sys_info[_inst_pkg] = reqs.decode().splitlines()

    else:
        jsond = json.loads(conda_out.decode())
        keys = {
            'conda_build_version': _conda_build_ver,
            'conda_env_version': _conda_env_ver,
            'platform': _conda_platform,
            'python_version': _conda_python_ver,
            'root_writable': _conda_root_writable,
        }
        for conda_k, sysinfo_k in keys.items():
            sys_info[sysinfo_k] = jsond.get(conda_k, 'N/A')

        # Get info about packages in current environment
        cmd = ('conda', 'list')
        try:
            conda_out = check_output(cmd)
        except CalledProcessError as e:
            _error_log.append(f'Error (conda): {e}')
        else:
            data = conda_out.decode().splitlines()
            sys_info[_inst_pkg] = [l for l in data if not l.startswith('#')]

    sys_info.update(get_os_spec_info(sys_info[_os_name]))
    sys_info[_errors] = _error_log
    sys_info[_warnings] = _warning_log
    sys_info[_runtime] = (datetime.now() - sys_info[_start]).total_seconds()
    return sys_info
예제 #32
0
def get_sys_info():
    # delay these imports until now as they are only needed in this
    # function which then exits.
    import platform
    import json
    import multiprocessing
    from numba import config
    from numba import cuda as cu
    from numba.cuda import cudadrv
    from numba.cuda.cudadrv.driver import driver as cudriver
    from numba import roc
    from numba.roc.hlc import hlc, libhlc
    import textwrap as tw
    import ctypes as ct
    import llvmlite.binding as llvmbind
    import locale
    from datetime import datetime
    from itertools import chain
    from subprocess import check_output, CalledProcessError

    try:
        fmt = "%-45s : %-s"
        print("-" * 80)
        print("__Time Stamp__")
        print(datetime.utcnow())
        print("")

        print("__Hardware Information__")
        system_name = platform.system()
        print(fmt % ("Machine", platform.machine()))
        print(fmt % ("CPU Name", llvmbind.get_host_cpu_name()))
        if system_name == 'Linux':
            strmatch = 'Cpus_allowed'
            try:
                loc = '/proc/self/status'
                with open(loc, 'rt') as f:
                    proc_stat = f.read().splitlines()
                    for x in proc_stat:
                        if x.startswith(strmatch):
                            if x.startswith('%s:' % strmatch):
                                hexnum = '0x%s' % x.split(':')[1].strip()
                                acc_cpus = int(hexnum, 16)
                                _n = str(bin(acc_cpus).count('1'))
                                print(fmt %
                                      ("Number of accessible CPU cores", _n))
                            elif x.startswith('%s_list:' % strmatch):
                                _a = x.split(':')[1].strip()
                                print(fmt %
                                      ("Listed accessible CPUs cores", _a))
            except Exception:
                print(fmt % ("CPU count", multiprocessing.cpu_count()))
            # See if CFS is in place
            # https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
            try:

                def scrape_lines(loc):
                    with open(loc, 'rt') as f:
                        return f.read().splitlines()

                loc = '/sys/fs/cgroup/cpuacct/cpu.cfs_period_us'
                cfs_period = int(scrape_lines(loc)[0])
                loc = '/sys/fs/cgroup/cpuacct/cpu.cfs_quota_us'
                cfs_quota = int(scrape_lines(loc)[0])
                if cfs_quota == -1:
                    print(fmt % ("CFS restrictions", "None"))
                else:
                    runtime_amount = float(cfs_quota) / float(cfs_period)
                    print(fmt % ("CFS restrictions (CPUs worth of runtime)",
                                 runtime_amount))
            except Exception:
                print(fmt % ("CFS restrictions", 'Information not available'))
        else:
            print(fmt % ("CPU count", multiprocessing.cpu_count()))

        try:
            featuremap = llvmbind.get_host_cpu_features()
        except RuntimeError:
            print(fmt % ("CPU Features", "NA"))
        else:
            features = sorted(
                [key for key, value in featuremap.items() if value])
            cpu_feat = tw.fill(' '.join(features), 80)
            print(fmt % ("CPU Features", ""))
            print(cpu_feat)
        print("")

        print("__OS Information__")
        print(fmt % ("Platform", platform.platform(aliased=True)))
        print(fmt % ("Release", platform.release()))
        print(fmt % ("System Name", system_name))
        print(fmt % ("Version", platform.version()))
        try:
            if system_name == 'Linux':
                info = platform.linux_distribution()
            elif system_name == 'Windows':
                info = platform.win32_ver()
            elif system_name == 'Darwin':
                info = platform.mac_ver()
            else:
                raise RuntimeError("Unknown system.")
            buf = ''.join([
                x if x != '' else ' ' for x in list(chain.from_iterable(info))
            ])
            print(fmt % ("OS specific info", buf))

            if system_name == 'Linux':
                print(fmt % ("glibc info", ' '.join(platform.libc_ver())))
        except:
            print("Error: System name incorrectly identified or unknown.")
        print("")

        print("__Python Information__")
        print(fmt % ("Python Compiler", platform.python_compiler()))
        print(fmt %
              ("Python Implementation", platform.python_implementation()))
        print(fmt % ("Python Version", platform.python_version()))
        lcl = []
        try:
            for x in locale.getdefaultlocale():
                if x is not None:
                    lcl.append(x)
        except Exception as e:
            lcl.append(str(e))
        print(fmt % ("Python Locale ", ' '.join(lcl)))

        print("")
        print("__LLVM information__")
        print(fmt % ("LLVM version", '.'.join(
            [str(k) for k in llvmbind.llvm_version_info])))

        print("")
        print("__CUDA Information__")
        # Look for GPUs
        try:
            cu.list_devices()[0]  # will a device initialise?
        except Exception as e:
            msg_not_found = "CUDA driver library cannot be found"
            msg_disabled_by_user = "******"
            msg_end = " or no CUDA enabled devices are present."
            msg_generic_problem = "Error: CUDA device intialisation problem."
            msg = getattr(e, 'msg', None)
            if msg is not None:
                if msg_not_found in msg:
                    err_msg = msg_not_found + msg_end
                elif msg_disabled_by_user in msg:
                    err_msg = msg_disabled_by_user + msg_end
                else:
                    err_msg = msg_generic_problem + " Message:" + msg
            else:
                err_msg = msg_generic_problem + " " + str(e)
            # Best effort error report
            print("%s\nError class: %s" % (err_msg, str(type(e))))
        else:
            try:
                cu.detect()
                dv = ct.c_int(0)
                cudriver.cuDriverGetVersion(ct.byref(dv))
                print(fmt % ("CUDA driver version", dv.value))
                print("CUDA libraries:")
                cudadrv.libs.test(sys.platform, print_paths=False)
            except:
                print(
                    "Error: Probing CUDA failed (device and driver present, runtime problem?)\n"
                )

        print("")
        print("__ROC Information__")
        roc_is_available = roc.is_available()
        print(fmt % ("ROC available", roc_is_available))

        toolchains = []
        try:
            libhlc.HLC()
            toolchains.append('librocmlite library')
        except:
            pass
        try:
            cmd = hlc.CmdLine().check_tooling()
            toolchains.append('ROC command line tools')
        except:
            pass

        # if no ROC try and report why
        if not roc_is_available:
            from numba.roc.hsadrv.driver import hsa
            try:
                hsa.is_available
            except Exception as e:
                msg = str(e)
            else:
                msg = 'No ROC toolchains found.'
            print(fmt % ("Error initialising ROC due to", msg))

        if toolchains:
            print(fmt % ("Available Toolchains", ', '.join(toolchains)))

        try:
            # ROC might not be available due to lack of tool chain, but HSA
            # agents may be listed
            from numba.roc.hsadrv.driver import hsa, dgpu_count
            decode = lambda x: x.decode('utf-8') if isinstance(x, bytes) else x
            print("\nFound %s HSA Agents:" % len(hsa.agents))
            for i, agent in enumerate(hsa.agents):
                print('Agent id  : %s' % i)
                print('    vendor: %s' % decode(agent.vendor_name))
                print('      name: %s' % decode(agent.name))
                print('      type: %s' % agent.device)
                print("")

            _dgpus = []
            for a in hsa.agents:
                if a.is_component and a.device == 'GPU':
                    _dgpus.append(decode(a.name))
            print(fmt % ("Found %s discrete GPU(s)" % dgpu_count(), \
                  ', '.join(_dgpus)))
        except Exception as e:
            print("No HSA Agents found, encountered exception when searching:")
            print(e)

        print("")
        print("__SVML Information__")
        # replicate some SVML detection logic from numba.__init__ here.
        # if SVML load fails in numba.__init__ the splitting of the logic
        # here will help diagnosis of the underlying issue
        have_svml_library = True
        try:
            if sys.platform.startswith('linux'):
                llvmbind.load_library_permanently("libsvml.so")
            elif sys.platform.startswith('darwin'):
                llvmbind.load_library_permanently("libsvml.dylib")
            elif sys.platform.startswith('win'):
                llvmbind.load_library_permanently("svml_dispmd")
            else:
                have_svml_library = False
        except:
            have_svml_library = False
        func = getattr(llvmbind.targets, "has_svml", None)
        llvm_svml_patched = func() if func is not None else False
        svml_operational = (config.USING_SVML and llvm_svml_patched \
                            and have_svml_library)
        print(fmt % ("SVML state, config.USING_SVML", config.USING_SVML))
        print(fmt % ("SVML library found and loaded", have_svml_library))
        print(fmt % ("llvmlite using SVML patched LLVM", llvm_svml_patched))
        print(fmt % ("SVML operational", svml_operational))

        # Check which threading backends are available.
        print("")
        print("__Threading Layer Information__")

        def parse_error(e, backend):
            # parses a linux based error message, this is to provide feedback
            # and hide user paths etc
            try:
                path, problem, symbol = [x.strip() for x in e.msg.split(':')]
                extn_dso = os.path.split(path)[1]
                if backend in extn_dso:
                    return "%s: %s" % (problem, symbol)
            except Exception:
                pass
            return "Unknown import problem."

        try:
            from numba.npyufunc import tbbpool
            print(fmt % ("TBB Threading layer available", True))
        except ImportError as e:
            # might be a missing symbol due to e.g. tbb libraries missing
            print(fmt % ("TBB Threading layer available", False))
            print(fmt % ("+--> Disabled due to", parse_error(e, 'tbbpool')))

        try:
            from numba.npyufunc import omppool
            print(fmt % ("OpenMP Threading layer available", True))
        except ImportError as e:
            print(fmt % ("OpenMP Threading layer available", False))
            print(fmt % ("+--> Disabled due to", parse_error(e, 'omppool')))

        try:
            from numba.npyufunc import workqueue
            print(fmt % ("Workqueue Threading layer available", True))
        except ImportError as e:
            print(fmt % ("Workqueue Threading layer available", False))
            print(fmt % ("+--> Disabled due to", parse_error(e, 'workqueue')))

        # look for numba env vars that are set
        print("")
        print("__Numba Environment Variable Information__")
        _envvar_found = False
        for k, v in os.environ.items():
            if k.startswith('NUMBA_'):
                print(fmt % (k, v))
                _envvar_found = True
        if not _envvar_found:
            print("None set.")

        # Look for conda and conda information
        print("")
        print("__Conda Information__")
        cmd = ["conda", "info", "--json"]
        try:
            conda_out = check_output(cmd)
        except Exception as e:
            print("Conda not present/not working.\nError was %s\n" % e)
        else:
            data = ''.join(conda_out.decode("utf-8").splitlines())
            jsond = json.loads(data)
            keys = [
                'conda_build_version', 'conda_env_version', 'platform',
                'python_version', 'root_writable'
            ]
            for k in keys:
                try:
                    print(fmt % (k, jsond[k]))
                except KeyError:
                    pass

            # get info about current environment
            cmd = ["conda", "list"]
            try:
                conda_out = check_output(cmd)
            except CalledProcessError as e:
                print("Error: Conda command failed. Error was %s\n" % e.output)
            else:
                print("")
                print("__Current Conda Env__")
                data = conda_out.decode("utf-8").splitlines()
                for k in data:
                    if k[0] != '#':  # don't show where the env is, personal data
                        print(k)

        print("-" * 80)

    except Exception as e:
        print("Error: The system reporting tool has failed unexpectedly.")
        print("Exception was:")
        print(e)

    finally:
        print(
            "%s" %
            "If requested, please copy and paste the information between\n"
            "the dashed (----) lines, or from a given specific section as\n"
            "appropriate.\n\n"
            "=============================================================\n"
            "IMPORTANT: Please ensure that you are happy with sharing the\n"
            "contents of the information present, any information that you\n"
            "wish to keep private you should remove before sharing.\n"
            "=============================================================\n")
예제 #33
0
def get_sysinfo():

    # Gather the information that shouldn't raise exceptions
    sys_info = {
        _start: datetime.now(),
        _start_utc: datetime.utcnow(),
        _machine: platform.machine(),
        _cpu_name: llvmbind.get_host_cpu_name(),
        _cpu_count: multiprocessing.cpu_count(),
        _platform_name: platform.platform(aliased=True),
        _platform_release: platform.release(),
        _os_name: platform.system(),
        _os_version: platform.version(),
        _python_comp: platform.python_compiler(),
        _python_impl: platform.python_implementation(),
        _python_version: platform.python_version(),
        _numba_env_vars: {k: v for (k, v) in os.environ.items()
                          if k.startswith('NUMBA_')},
        _numba_version: version_number,
        _llvm_version: '.'.join(str(i) for i in llvmbind.llvm_version_info),
        _llvmlite_version: llvmlite_version,
        _roc_available: roc.is_available(),
        _psutil: _psutil_import,
    }

    # CPU features
    try:
        feature_map = llvmbind.get_host_cpu_features()
    except RuntimeError as e:
        _error_log.append(f'Error (CPU features): {e}')
    else:
        features = sorted([key for key, value in feature_map.items() if value])
        sys_info[_cpu_features] = ' '.join(features)

    # Python locale
    # On MacOSX, getdefaultlocale can raise. Check again if Py > 3.7.5
    try:
        # If $LANG is unset, getdefaultlocale() can return (None, None), make
        # sure we can encode this as strings by casting explicitly.
        sys_info[_python_locale] = '.'.join([str(i) for i in
                                             locale.getdefaultlocale()])
    except Exception as e:
        _error_log.append(f'Error (locale): {e}')

    # CUDA information
    try:
        cu.list_devices()[0]  # will a device initialise?
    except Exception as e:
        sys_info[_cu_dev_init] = False
        msg_not_found = "CUDA driver library cannot be found"
        msg_disabled_by_user = "******"
        msg_end = " or no CUDA enabled devices are present."
        msg_generic_problem = "CUDA device intialisation problem."
        msg = getattr(e, 'msg', None)
        if msg is not None:
            if msg_not_found in msg:
                err_msg = msg_not_found + msg_end
            elif msg_disabled_by_user in msg:
                err_msg = msg_disabled_by_user + msg_end
            else:
                err_msg = msg_generic_problem + " Message:" + msg
        else:
            err_msg = msg_generic_problem + " " + str(e)
        # Best effort error report
        _warning_log.append("Warning (cuda): %s\nException class: %s" %
                            (err_msg, str(type(e))))
    else:
        try:
            sys_info[_cu_dev_init] = True

            output = StringIO()
            with redirect_stdout(output):
                cu.detect()
            sys_info[_cu_detect_out] = output.getvalue()
            output.close()

            dv = ctypes.c_int(0)
            cudriver.cuDriverGetVersion(ctypes.byref(dv))
            sys_info[_cu_drv_ver] = dv.value

            rtver = ctypes.c_int(0)
            curuntime.cudaRuntimeGetVersion(ctypes.byref(rtver))
            sys_info[_cu_rt_ver] = rtver.value

            output = StringIO()
            with redirect_stdout(output):
                cudadrv.libs.test(sys.platform, print_paths=False)
            sys_info[_cu_lib_test] = output.getvalue()
            output.close()
        except Exception as e:
            _warning_log.append(
                "Warning (cuda): Probing CUDA failed "
                "(device and driver present, runtime problem?)\n"
                f"(cuda) {type(e)}: {e}")

    # ROC information
    # If no ROC try and report why
    if not sys_info[_roc_available]:
        from numba.roc.hsadrv.driver import hsa
        try:
            hsa.is_available
        except Exception as e:
            msg = str(e)
        else:
            msg = 'No ROC toolchains found.'
        _warning_log.append(f"Warning (roc): Error initialising ROC: {msg}")

    toolchains = []
    try:
        libhlc.HLC()
        toolchains.append('librocmlite library')
    except Exception:
        pass
    try:
        cmd = hlc.CmdLine().check_tooling()
        toolchains.append('ROC command line tools')
    except Exception:
        pass
    sys_info[_roc_toolchains] = toolchains

    try:
        # ROC might not be available due to lack of tool chain, but HSA
        # agents may be listed
        from numba.roc.hsadrv.driver import hsa, dgpu_count

        def decode(x):
            return x.decode('utf-8') if isinstance(x, bytes) else x

        sys_info[_hsa_agents_count] = len(hsa.agents)
        agents = []
        for i, agent in enumerate(hsa.agents):
            agents.append({
                'Agent id': i,
                'Vendor': decode(agent.vendor_name),
                'Name': decode(agent.name),
                'Type': agent.device,
            })
        sys_info[_hsa_agents] = agents

        _dgpus = []
        for a in hsa.agents:
            if a.is_component and a.device == 'GPU':
                _dgpus.append(decode(a.name))
        sys_info[_hsa_gpus_count] = dgpu_count()
        sys_info[_hsa_gpus] = ', '.join(_dgpus)
    except Exception as e:
        _warning_log.append(
            "Warning (roc): No HSA Agents found, "
            f"encountered exception when searching: {e}")

    # SVML information
    # Replicate some SVML detection logic from numba.__init__ here.
    # If SVML load fails in numba.__init__ the splitting of the logic
    # here will help diagnosing the underlying issue.
    svml_lib_loaded = True
    try:
        if sys.platform.startswith('linux'):
            llvmbind.load_library_permanently("libsvml.so")
        elif sys.platform.startswith('darwin'):
            llvmbind.load_library_permanently("libsvml.dylib")
        elif sys.platform.startswith('win'):
            llvmbind.load_library_permanently("svml_dispmd")
        else:
            svml_lib_loaded = False
    except Exception:
        svml_lib_loaded = False
    func = getattr(llvmbind.targets, "has_svml", None)
    sys_info[_llvm_svml_patched] = func() if func else False
    sys_info[_svml_state] = config.USING_SVML
    sys_info[_svml_loaded] = svml_lib_loaded
    sys_info[_svml_operational] = all((
        sys_info[_svml_state],
        sys_info[_svml_loaded],
        sys_info[_llvm_svml_patched],
    ))

    # Check which threading backends are available.
    def parse_error(e, backend):
        # parses a linux based error message, this is to provide feedback
        # and hide user paths etc
        try:
            path, problem, symbol = [x.strip() for x in e.msg.split(':')]
            extn_dso = os.path.split(path)[1]
            if backend in extn_dso:
                return "%s: %s" % (problem, symbol)
        except Exception:
            pass
        return "Unknown import problem."

    try:
        from numba.np.ufunc import tbbpool  # NOQA
        sys_info[_tbb_thread] = True
    except ImportError as e:
        # might be a missing symbol due to e.g. tbb libraries missing
        sys_info[_tbb_thread] = False
        sys_info[_tbb_error] = parse_error(e, 'tbbpool')

    try:
        from numba.np.ufunc import omppool
        sys_info[_openmp_thread] = True
        sys_info[_openmp_vendor] = omppool.openmp_vendor
    except ImportError as e:
        sys_info[_openmp_thread] = False
        sys_info[_openmp_error] = parse_error(e, 'omppool')

    try:
        from numba.np.ufunc import workqueue  # NOQA
        sys_info[_wkq_thread] = True
    except ImportError as e:
        sys_info[_wkq_thread] = True
        sys_info[_wkq_error] = parse_error(e, 'workqueue')

    # Look for conda and installed packages information
    cmd = ('conda', 'info', '--json')
    try:
        conda_out = check_output(cmd)
    except Exception as e:
        _warning_log.append(f'Warning: Conda not available.\n Error was {e}\n')
        # Conda is not available, try pip list to list installed packages
        cmd = (sys.executable, '-m', 'pip', 'list')
        try:
            reqs = check_output(cmd)
        except Exception as e:
            _error_log.append(f'Error (pip): {e}')
        else:
            sys_info[_inst_pkg] = reqs.decode().splitlines()

    else:
        jsond = json.loads(conda_out.decode())
        keys = {
            'conda_build_version': _conda_build_ver,
            'conda_env_version': _conda_env_ver,
            'platform': _conda_platform,
            'python_version': _conda_python_ver,
            'root_writable': _conda_root_writable,
        }
        for conda_k, sysinfo_k in keys.items():
            sys_info[sysinfo_k] = jsond.get(conda_k, 'N/A')

        # Get info about packages in current environment
        cmd = ('conda', 'list')
        try:
            conda_out = check_output(cmd)
        except CalledProcessError as e:
            _error_log.append(f'Error (conda): {e}')
        else:
            data = conda_out.decode().splitlines()
            sys_info[_inst_pkg] = [l for l in data if not l.startswith('#')]

    sys_info.update(get_os_spec_info(sys_info[_os_name]))
    sys_info[_errors] = _error_log
    sys_info[_warnings] = _warning_log
    sys_info[_runtime] = (datetime.now() - sys_info[_start]).total_seconds()
    return sys_info
예제 #34
0
def detect_cuda_driver() -> bool:
    try:
        return len(cuda.list_devices()) > 0
    except cuda.CudaSupportError:
        return False
예제 #35
0
from __future__ import print_function

import threading
from math import ceil

import numpy as np

from numba import cuda

print('System has %d CUDA devices' % len(cuda.list_devices()))

signature = 'void(int32[:], int32[:])'

def kernel(dst, src):
    '''A simple kernel that adds 1 to every item
    '''
    i = cuda.grid(1)
    if i >= dst.shape[0]:
        return
    dst[i] = src[i] + 1

# Numba compiler is not threadsafe
compiler_lock = threading.Lock()

def device_controller(cid):
    cuda.select_device(cid)                    # bind device to thread
    device = cuda.get_current_device()         # get current device

    # print some information about the CUDA card
    prefix = '[%s]' % device
    print(prefix, 'device_controller', cid, '| CC', device.COMPUTE_CAPABILITY)
예제 #36
0
def main():
    parser = parseArgs()
    print 'Reading in file:', parser.xml_file
    data = openWithAse(parser.xml_file)
    print 'Done.'
    h5_file = createH5File(parser, data)
    workers = multiprocessing.cpu_count()
    batch_size = workers
    n_batches = len(data) / batch_size + 1
    gpu_ids = range(len(cuda.list_devices()))
    if parser.gpu:
        batch_size /= len(cuda.list_devices())
        total = 0
        n_batches = len(data) / (batch_size) + 1
        outer_pool = ThreadPool(len(cuda.list_devices()))

        for batch in range(0, n_batches, len(gpu_ids)):
            start = batch
            end = batch + len(gpu_ids)
            if end > n_batches:
                end = n_batches

            batches = range(start, end)
            batch_sizes = [batch_size] * len(batches)
            data_lens = [len(data)] * len(batches)
            results = outer_pool.map(
                gpuWork,
                zip(batches, batch_sizes, gpu_ids, data_lens,
                    [data] * len(batches), [parser] * len(batches)))
            for i, (x, y) in enumerate(results):
                write_start = total * batch_size
                write_stop = (total + 1) * batch_size
                if write_stop > len(data):
                    write_stop = len(data)
                h5_file['X'][write_start:write_stop] = x.reshape(x.shape +
                                                                 (1, ))
                h5_file['Y'][write_start:write_stop] = y.reshape((len(y), 1))
                total += 1
            sys.stdout.write('Progress: %0.2f%%\r' %
                             (100. * batch / n_batches))
            sys.stdout.flush()
    else:
        pool = Pool(workers)
        for i in bar(range(n_batches)):
            start = i * batch_size
            end = (i + 1) * batch_size
            if end > len(data):
                end = len(data)
            results = pool.map(
                randRotateAndTranslateWrap,
                zip([parser] * (end - start), data[start:end], [
                    0,
                ] * (end - start), [True] * (end - start)))
            images, energies = [], []
            for im, e in results:
                images.append(im)
                energies.append(e)

            images = np.array(images, dtype=np.float32)
            energies = np.array(energies, dtype=np.float32)
            h5_file['X'][start:end] = images.reshape(images.shape + (1, ))
            h5_file['Y'][start:end] = energies.reshape((len(energies), 1))
            # sys.stdout.write('Progress: %0.2f%%\r' % (100. * i / n_batches))
            # sys.stdout.flush()
    # for i in range(0, len(h5_file['Y']), 100):
    #    plt.imshow(h5_file['X'][i].reshape((256, 256)))
    #    plt.show()
    h5_file.attrs['cell_dimensions'] = data[0].cell
    h5_file.attrs['min'] = np.min(h5_file['Y'][:])
    h5_file.attrs['max'] = np.max(h5_file['Y'][:])
    h5_file.close()
예제 #37
0
  booktitle =    {5th International Conference on Computational Social Science},
  year =         {2019},
  url =          {http://app.ic2s2.org/app/sessions/9kXqn5btgKKC5yfCvg/details}
}
"""
import os, sys
import numpy as np
from numba import cuda
import multiprocessing

start_time = input("Run start time (in \"%Y-%m-%dT%H:%M:%S%fZ\" format: ")
ticks_needed = input("Number of hours to run: ")
mmd_range = eval(input("MAX MEMORY DEPTH RANGE (in (min,max,step) format: "))
alpha_range = eval(input("ALPHA RANGE (in (min,max,step) format: "))
reps = int(eval(input("Number of repetitions per experiment: ")))

python_command = os.path.basename(sys.executable)

mmd_range = eval("range{0}".format(mmd_range))
alpha_range = eval("np.arange{0}".format(alpha_range))
total_runs = len(mmd_range) * len(alpha_range)
num_devices = len(cuda.list_devices())
i = 0

for rep in range(reps):
    for mmd in mmd_range:
        for alpha in alpha_range:
            print("Running Rep={2}, MMD={0}, alpha={1}".format(
                mmd, alpha, rep))
            os.system("{0} MACM.py {1} {2} {3} {4} --quiet ".format(
                python_command, start_time, ticks_needed, mmd, alpha))