def compile_gpu_func(nan_is_error, inf_is_error, big_is_error): """ compile utility function used by contains_nan and contains_inf """ global f_gpumin, f_gpumax, f_gpuabsmax if not cuda.cuda_available: return guard_input = cuda.fvector("nan_guard") cuda_compile_failed = False if (nan_is_error or inf_is_error) and f_gpumin is None: try: f_gpumin = theano.function([guard_input], T.min(guard_input), mode="FAST_RUN") except RuntimeError: # This can happen if cuda is available, but the # device is in exclusive mode and used by another # process. cuda_compile_failed = True if inf_is_error and not cuda_compile_failed and f_gpumax is None: try: f_gpumax = theano.function([guard_input], T.max(guard_input), mode="FAST_RUN") except RuntimeError: # This can happen if cuda is available, but the # device is in exclusive mode and used by another # process. cuda_compile_failed = True if big_is_error and not cuda_compile_failed and f_gpuabsmax is None: try: f_gpuabsmax = theano.function([guard_input], T.max(T.abs_(guard_input)), mode="FAST_RUN") except RuntimeError: # This can happen if cuda is available, but the # device is in exclusive mode and used by another # process. cuda_compile_failed = True
def test_output_broadcast_cuda(self): from theano.sandbox import cuda if not cuda.cuda_available: raise SkipTest("Optional package Cuda disabled") v = cuda.fvector('v') c, r = VecAsRowAndCol()(v) f = theano.function([v], [c, r]) v_val = cuda.CudaNdarray(self.rng.randn(5).astype('float32')) f(v_val)
def test_nvidia_driver3(): """ Test that the gpu device is initialized by theano when we build a function with gpu op. The driver should always be tested during theano initialization of the gpu device """ var = cuda.fvector() f = theano.function([var], var + 1, mode=mode_with_gpu) topo = f.maker.env.toposort() assert any([isinstance(node.op, cuda.GpuElemwise) for node in topo]) assert theano.sandbox.cuda.use.device_number is not None
def test_nvidia_driver3(): """ Test that the gpu device is initialized by theano when we build a function with gpu op. The driver should always be tested during theano initialization of the gpu device """ var = cuda.fvector() f = theano.function([var], var + 1, mode=mode_with_gpu, profile=False) topo = f.maker.fgraph.toposort() assert any([isinstance(node.op, cuda.GpuElemwise) for node in topo]) assert theano.sandbox.cuda.use.device_number is not None
def test_output_broadcast_cuda(self): from theano.sandbox import cuda if not cuda.cuda_available: raise SkipTest("Optional package Cuda disabled") if cuda.use.device_number is None: # We should normally set VecAsRowAndCol as a GPUOp But we # don't want to do this here as this will disable others # tests in this file. So we manually init the GPU if # needed to remove warning. cuda.use("gpu", force=True, default_to_move_computation_to_gpu=False, move_shared_float32_to_gpu=False, enable_cuda=False) v = cuda.fvector('v') c, r = VecAsRowAndCol()(v) f = theano.function([v], [c, r]) v_val = cuda.CudaNdarray(self.rng.randn(5).astype('float32')) f(v_val)
def compile_gpu_func(nan_is_error, inf_is_error, big_is_error): """ compile utility function used by contains_nan and contains_inf """ global f_gpumin, f_gpumax, f_gpuabsmax if not cuda.cuda_available: return guard_input = cuda.fvector('nan_guard') cuda_compile_failed = False if (nan_is_error or inf_is_error) and f_gpumin is None: try: f_gpumin = theano.function( [guard_input], T.min(guard_input), mode='FAST_RUN' ) except RuntimeError: # This can happen if cuda is available, but the # device is in exclusive mode and used by another # process. cuda_compile_failed = True if inf_is_error and not cuda_compile_failed and f_gpumax is None: try: f_gpumax = theano.function( [guard_input], T.max(guard_input), mode='FAST_RUN' ) except RuntimeError: # This can happen if cuda is available, but the # device is in exclusive mode and used by another # process. cuda_compile_failed = True if big_is_error and not cuda_compile_failed and f_gpuabsmax is None: try: f_gpuabsmax = theano.function( [guard_input], T.max(T.abs_(guard_input)), mode='FAST_RUN' ) except RuntimeError: # This can happen if cuda is available, but the # device is in exclusive mode and used by another # process. cuda_compile_failed = True
def test_vector(self): x = cuda.fvector() y = numpy.zeros(7, dtype='float32') assert y.size == theano.function([x], x.size)(y)
def __init__(self, nan_is_error, inf_is_error, big_is_error=True): if cuda.cuda_available: self.guard_input = cuda.fvector('nan_guard') if nan_is_error or inf_is_error: self.gpumin = theano.function( [self.guard_input], T.min(self.guard_input), mode='FAST_RUN' ) if inf_is_error: self.gpumax = theano.function( [self.guard_input], T.max(self.guard_input), mode='FAST_RUN' ) if big_is_error: self.gpuabsmax = theano.function( [self.guard_input], T.max(T.abs_(self.guard_input)), mode='FAST_RUN' ) def do_check_on(var, nd, f, is_input): """ Checks `var` for NaNs / Infs. If detected, raises an exception and / or prints information about `nd`, `f`, and `is_input` to help the user determine the cause of the invalid values. Parameters ---------- var : numpy.ndarray The value to be checked. nd : theano.gof.Apply The Apply node being executed f : callable The thunk for the apply node is_input : bool If True, `var` is an input to `nd`. If False, it is an output. """ error = False if nan_is_error: err = False if cuda.cuda_available and isinstance(var, cuda.CudaNdarray): err = np.isnan(self.gpumin(var.reshape(var.size))) else: err = contains_nan(var) if err: logger.error('NaN detected') error = True if inf_is_error: err = False if cuda.cuda_available and isinstance(var, cuda.CudaNdarray): err = (np.isinf(self.gpumin(var.reshape(var.size))) or np.isinf(self.gpumax(var.reshape(var.size)))) else: err = contains_inf(var) if err: logger.error('Inf detected') error = True if big_is_error: err = False if cuda.cuda_available and isinstance(var, cuda.CudaNdarray): err = (self.gpuabsmax(var.reshape(var.size)) > 1e10) elif isinstance(var, theano.gof.type.CDataType._cdata_type): err = False elif isinstance(var, np.random.mtrand.RandomState): err = False else: err = (np.abs(var).max() > 1e10) if err: logger.error('Big value detected') error = True if error: if is_input: logger.error('In an input') else: logger.error('In an output') logger.error('Inputs: ') for ivar, ival in zip(nd.inputs, f.inputs): logger.error('var') logger.error(ivar) logger.error(theano.printing.min_informative_str(ivar)) logger.error('val') logger.error(ival) logger.error('Node:') logger.error(nd) assert False def nan_check(i, node, fn): """ Runs `fn` while checking its inputs and outputs for NaNs / Infs Parameters ---------- i : currently ignored (TODO: determine why it is here or remove) node : theano.gof.Apply The Apply node currently being executed fn : callable The thunk to execute for this Apply node """ inputs = fn.inputs # TODO: figure out why individual inputs are themselves lists # sometimes for x in flatten(inputs): do_check_on(x, node, fn, True) fn() outputs = fn.outputs for j, x in enumerate(flatten(outputs)): do_check_on(x, node, fn, False) wrap_linker = theano.gof.WrapLinker([theano.gof.OpWiseCLinker()], nan_check) super(NanGuardMode, self).__init__(wrap_linker, optimizer=theano.config.optimizer)
def __init__(self, nan_is_error, inf_is_error, big_is_error=True): if cuda.cuda_available: self.guard_input = cuda.fvector('nan_guard') if nan_is_error or inf_is_error: self.gpumin = theano.function([self.guard_input], T.min(self.guard_input), mode='FAST_RUN') if inf_is_error: self.gpumax = theano.function([self.guard_input], T.max(self.guard_input), mode='FAST_RUN') if big_is_error: self.gpuabsmax = theano.function([self.guard_input], T.max(T.abs_( self.guard_input)), mode='FAST_RUN') def do_check_on(var, nd, f, is_input): """ Checks `var` for NaNs / Infs. If detected, raises an exception and / or prints information about `nd`, `f`, and `is_input` to help the user determine the cause of the invalid values. Parameters ---------- var : numpy.ndarray The value to be checked. nd : theano.gof.Apply The Apply node being executed f : callable The thunk for the apply node is_input : bool If True, `var` is an input to `nd`. If False, it is an output. """ error = False if nan_is_error: err = False if cuda.cuda_available and isinstance(var, cuda.CudaNdarray): err = np.isnan(self.gpumin(var.reshape(var.size))) else: err = contains_nan(var) if err: logger.error('NaN detected') error = True if inf_is_error: err = False if cuda.cuda_available and isinstance(var, cuda.CudaNdarray): err = (np.isinf(self.gpumin(var.reshape(var.size))) or \ np.isinf(self.gpumax(var.reshape(var.size)))) else: err = contains_inf(var) if err: logger.error('Inf detected') error = True if big_is_error: err = False if cuda.cuda_available and isinstance(var, cuda.CudaNdarray): err = (self.gpuabsmax(var.reshape(var.size)) > 1e10) else: err = (np.abs(var).max() > 1e10) if err: logger.error('Big value detected') error = True if error: if is_input: logger.error('In an input') else: logger.error('In an output') logger.error('Inputs: ') for ivar, ival in zip(nd.inputs, f.inputs): logger.error('var') logger.error(ivar) logger.error(theano.printing.min_informative_str(ivar)) logger.error('val') logger.error(ival) logger.error('Node:') logger.error(nd) assert False def nan_check(i, node, fn): """ Runs `fn` while checking its inputs and outputs for NaNs / Infs Parameters ---------- i : currently ignored (TODO: determine why it is here or remove) node : theano.gof.Apply The Apply node currently being executed fn : callable The thunk to execute for this Apply node """ inputs = fn.inputs # TODO: figure out why individual inputs are themselves lists sometimes for x in flatten(inputs): do_check_on(x, node, fn, True) fn() outputs = fn.outputs for j, x in enumerate(flatten(outputs)): do_check_on(x, node, fn, False) wrap_linker = theano.gof.WrapLinkerMany([theano.gof.OpWiseCLinker()], [nan_check]) super(NanGuardMode, self).__init__(wrap_linker, optimizer=theano.config.optimizer)
def __init__(self, nan_is_error=None, inf_is_error=None, big_is_error=None, optimizer=None, linker=None): self.provided_optimizer = optimizer cuda_compile_failed = False if nan_is_error is None: nan_is_error = config.NanGuardMode.nan_is_error if inf_is_error is None: inf_is_error = config.NanGuardMode.inf_is_error if big_is_error is None: big_is_error = config.NanGuardMode.big_is_error assert nan_is_error or inf_is_error or big_is_error if cuda.cuda_available: self.guard_input = cuda.fvector('nan_guard') if nan_is_error or inf_is_error: try: self.gpumin = theano.function([self.guard_input], T.min(self.guard_input), mode='FAST_RUN') except RuntimeError: # This can happen if cuda is available, but the # device is in exclusive mode and used by another # process. cuda_compile_failed = True if inf_is_error and not cuda_compile_failed: try: self.gpumax = theano.function([self.guard_input], T.max(self.guard_input), mode='FAST_RUN') except RuntimeError: # This can happen if cuda is available, but the # device is in exclusive mode and used by another # process. cuda_compile_failed = True if big_is_error and not cuda_compile_failed: try: self.gpuabsmax = theano.function( [self.guard_input], T.max(T.abs_(self.guard_input)), mode='FAST_RUN') except RuntimeError: # This can happen if cuda is available, but the # device is in exclusive mode and used by another # process. cuda_compile_failed = True def do_check_on(var, nd, f, is_input): """ Checks `var` for NaNs / Infs. If detected, raises an exception and / or prints information about `nd`, `f`, and `is_input` to help the user determine the cause of the invalid values. Parameters ---------- var : numpy.ndarray The value to be checked. nd : theano.gof.Apply The Apply node being executed. f : callable The thunk for the apply node. is_input : bool If True, `var` is an input to `nd`. If False, it is an output. """ error = False if nan_is_error: err = False if cuda.cuda_available and isinstance(var, cuda.CudaNdarray): if not isinstance( nd.op, # It store ints in float container theano.sandbox.rng_mrg.GPU_mrg_uniform): err = np.isnan(self.gpumin(var.reshape(var.size))) else: err = contains_nan(var) if err: logger.error('NaN detected') error = True if inf_is_error: err = False if cuda.cuda_available and isinstance(var, cuda.CudaNdarray): err = (np.isinf(self.gpumin(var.reshape(var.size))) or np.isinf(self.gpumax(var.reshape(var.size)))) else: err = contains_inf(var) if err: logger.error('Inf detected') error = True if big_is_error: err = False if cuda.cuda_available and isinstance(var, cuda.CudaNdarray): err = (self.gpuabsmax(var.reshape(var.size)) > 1e10) elif isinstance(var, theano.gof.type.CDataType._cdata_type): err = False elif isinstance(var, np.random.mtrand.RandomState): err = False else: err = (np.abs(var).max() > 1e10) if err: logger.error('Big value detected') error = True if error: if is_input: logger.error('In an input') else: logger.error('In an output') logger.error('Inputs: ') for ivar, ival in zip(nd.inputs, f.inputs): logger.error('var') logger.error(ivar) logger.error(theano.printing.min_informative_str(ivar)) logger.error('val') logger.error(ival) logger.error('Node:') logger.error(nd) assert False def nan_check(i, node, fn): """ Runs `fn` while checking its inputs and outputs for NaNs / Infs. Parameters ---------- i : Currently ignored. TODO: determine why it is here or remove). node : theano.gof.Apply The Apply node currently being executed. fn : callable The thunk to execute for this Apply node. """ inputs = fn.inputs # TODO: figure out why individual inputs are themselves lists # sometimes for x in flatten(inputs): do_check_on(x, node, fn, True) fn() outputs = fn.outputs for j, x in enumerate(flatten(outputs)): do_check_on(x, node, fn, False) wrap_linker = theano.gof.WrapLinker([theano.gof.OpWiseCLinker()], nan_check) super(NanGuardMode, self).__init__(wrap_linker, optimizer=self.provided_optimizer)