示例#1
0
文件: grid.py 项目: phobson/pygridgen
    def mask_rho(self):
        """
        Returns the mask for the cells
        """
        if self._mask_rho is None:
            mask_shape = tuple([n - 1 for n in self.x_vert.shape])
            self._mask_rho = numpy.ones(mask_shape, dtype='d')

            # If maskedarray is given for vertices, modify the mask such that
            # non-existant grid points are masked.  A cell requires all four
            # verticies to be defined as a water point.
            if isinstance(self.x_vert, numpy.ma.MaskedArray):
                mask = (self.x_vert.mask[:-1, :-1] | self.x_vert.mask[1:, :-1] |
                        self.x_vert.mask[:-1, 1:] | self.x_vert.mask[1:, 1:])
                self._mask_rho = numpy.asarray(
                    ~(~numpy.bool_(self.mask_rho) | mask),
                    dtype='d'
                )

            if isinstance(self.y_vert, numpy.ma.MaskedArray):
                mask = (self.y_vert.mask[:-1, :-1] | self.y_vert.mask[1:, :-1] |
                        self.y_vert.mask[:-1, 1:] | self.y_vert.mask[1:, 1:])
                self._mask_rho = numpy.asarray(
                    ~(~numpy.bool_(self.mask_rho) | mask),
                    dtype='d'
                )

        return self._mask_rho
示例#2
0
文件: grid.py 项目: jingzhiyou/octant
 def __init__(self, x, y):
             
     assert np.ndim(x)==2 and np.ndim(y)==2 and np.shape(x)==np.shape(y), \
         'x and y must be 2D arrays of the same size.'
     
     if np.any(np.isnan(x)) or np.any(np.isnan(y)):
         x = np.ma.masked_where( (isnan(x)) | (isnan(y)) , x)
         y = np.ma.masked_where( (isnan(x)) | (isnan(y)) , y)
         
     self.x_vert = x
     self.y_vert = y
     
     mask_shape = tuple([n-1 for n in self.x_vert.shape])
     self.mask_rho = np.ones(mask_shape, dtype='d')
     
     # If maskedarray is given for verticies, modify the mask such that 
     # non-existant grid points are masked.  A cell requires all four
     # verticies to be defined as a water point.
     if isinstance(self.x_vert, np.ma.MaskedArray):
         mask = (self.x_vert.mask[:-1,:-1] | self.x_vert.mask[1:,:-1] | \
                 self.x_vert.mask[:-1,1:] | self.x_vert.mask[1:,1:])
         self.mask_rho = np.asarray(~(~np.bool_(self.mask_rho) | mask), dtype='d')
     
     if isinstance(self.y_vert, np.ma.MaskedArray):
         mask = (self.y_vert.mask[:-1,:-1] | self.y_vert.mask[1:,:-1] | \
                 self.y_vert.mask[:-1,1:] | self.y_vert.mask[1:,1:])
         self.mask_rho = np.asarray(~(~np.bool_(self.mask_rho) | mask), dtype='d')
     
     self._calculate_subgrids()
     self._calculate_metrics()        
示例#3
0
    def test00b_setBoolAttributes(self):
        """Checking setting Bool attributes (scalar, NumPy case)"""

        self.array.attrs.pq = numpy.bool_(True)
        self.array.attrs.qr = numpy.bool_(False)
        self.array.attrs.rs = numpy.bool_(True)

        # Check the results
        if common.verbose:
            print "pq -->", self.array.attrs.pq
            print "qr -->", self.array.attrs.qr
            print "rs -->", self.array.attrs.rs

        if self.close:
            if common.verbose:
                print "(closing file version)"
            self.fileh.close()
            self.fileh = openFile(self.file, mode = "r+")
            self.root = self.fileh.root
            self.array = self.fileh.root.anarray

        assert isinstance(self.root.anarray.attrs.pq, numpy.bool_)
        assert isinstance(self.root.anarray.attrs.qr, numpy.bool_)
        assert isinstance(self.root.anarray.attrs.rs, numpy.bool_)
        assert self.root.anarray.attrs.pq == True
        assert self.root.anarray.attrs.qr == False
        assert self.root.anarray.attrs.rs == True
示例#4
0
    def test_numpy(self):
        assert chash(np.bool_(True)) == chash(np.bool_(True))

        assert chash(np.int8(1)) == chash(np.int8(1))
        assert chash(np.int16(1))
        assert chash(np.int32(1))
        assert chash(np.int64(1))

        assert chash(np.uint8(1))
        assert chash(np.uint16(1))
        assert chash(np.uint32(1))
        assert chash(np.uint64(1))

        assert chash(np.float32(1)) == chash(np.float32(1))
        assert chash(np.float64(1)) == chash(np.float64(1))
        assert chash(np.float128(1)) == chash(np.float128(1))

        assert chash(np.complex64(1+1j)) == chash(np.complex64(1+1j))
        assert chash(np.complex128(1+1j)) == chash(np.complex128(1+1j))
        assert chash(np.complex256(1+1j)) == chash(np.complex256(1+1j))

        assert chash(np.datetime64('2000-01-01')) == chash(np.datetime64('2000-01-01'))
        assert chash(np.timedelta64(1,'W')) == chash(np.timedelta64(1,'W'))

        self.assertRaises(ValueError, chash, np.object())

        assert chash(np.array([[1, 2], [3, 4]])) == \
            chash(np.array([[1, 2], [3, 4]]))
        assert chash(np.array([[1, 2], [3, 4]])) != \
            chash(np.array([[1, 2], [3, 4]]).T)
        assert chash(np.array([1, 2, 3])) == chash(np.array([1, 2, 3]))
        assert chash(np.array([1, 2, 3], dtype=np.int32)) != \
            chash(np.array([1, 2, 3], dtype=np.int64))
示例#5
0
 def test_numpy_scalar_conversion_values(self):
     self.assertEqual(nd.as_py(nd.array(np.bool_(True))), True)
     self.assertEqual(nd.as_py(nd.array(np.bool_(False))), False)
     self.assertEqual(nd.as_py(nd.array(np.int8(100))), 100)
     self.assertEqual(nd.as_py(nd.array(np.int8(-100))), -100)
     self.assertEqual(nd.as_py(nd.array(np.int16(20000))), 20000)
     self.assertEqual(nd.as_py(nd.array(np.int16(-20000))), -20000)
     self.assertEqual(nd.as_py(nd.array(np.int32(1000000000))), 1000000000)
     self.assertEqual(nd.as_py(nd.array(np.int64(-1000000000000))),
                      -1000000000000)
     self.assertEqual(nd.as_py(nd.array(np.int64(1000000000000))),
                      1000000000000)
     self.assertEqual(nd.as_py(nd.array(np.int32(-1000000000))),
                      -1000000000)
     self.assertEqual(nd.as_py(nd.array(np.uint8(200))), 200)
     self.assertEqual(nd.as_py(nd.array(np.uint16(50000))), 50000)
     self.assertEqual(nd.as_py(nd.array(np.uint32(3000000000))), 3000000000)
     self.assertEqual(nd.as_py(nd.array(np.uint64(10000000000000000000))),
                      10000000000000000000)
     self.assertEqual(nd.as_py(nd.array(np.float32(2.5))), 2.5)
     self.assertEqual(nd.as_py(nd.array(np.float64(2.5))), 2.5)
     self.assertEqual(nd.as_py(nd.array(np.complex64(2.5-1j))), 2.5-1j)
     self.assertEqual(nd.as_py(nd.array(np.complex128(2.5-1j))), 2.5-1j)
     if np.__version__ >= '1.7':
         self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13'))),
                          date(2000, 12, 13))
示例#6
0
 def test_numpy_scalar_bool(self):
     x = np.bool_(True)
     x_rec = self.encode_decode(x)
     assert x == x_rec and type(x) == type(x_rec)
     x = np.bool_(False)
     x_rec = self.encode_decode(x)
     assert x == x_rec and type(x) == type(x_rec)
示例#7
0
    def eval(self, row, dataset):
        result = np.bool_(self.value[0].eval(row, dataset))

        for oper, val in self.operator_operands(self.value[1:]):
            val = np.bool_(val.eval(row, dataset))
            result = self.operation(oper, result, val)

        return result
示例#8
0
def test_normalize_json():
    doc = {"foo": {numpy.bool_(True): "value"},
           "what": numpy.bool_(False),
           "this": numpy.PINF}
    normalized_doc = normalize_json(doc)
    assert isinstance(normalized_doc['what'], bool)
    assert isinstance(list(normalized_doc['foo'].keys())[0], bool)
    assert normalized_doc['this'] == "Infinity"
示例#9
0
    def test_scalar_bool(self):
        x = np.bool_(1)
        x_rec = self.encode_decode(x)
        tm.assert_almost_equal(x, x_rec)

        x = np.bool_(0)
        x_rec = self.encode_decode(x)
        tm.assert_almost_equal(x, x_rec)
示例#10
0
def test_numpy_bool():
    import numpy as np
    convert, noconvert = m.bool_passthrough, m.bool_passthrough_noconvert

    # np.bool_ is not considered implicit
    assert convert(np.bool_(True)) is True
    assert convert(np.bool_(False)) is False
    assert noconvert(np.bool_(True)) is True
    assert noconvert(np.bool_(False)) is False
示例#11
0
 def test_numpy_scalar_bool(self):
     x = np.bool_(True)
     x_rec = self.encode_decode(x)
     assert_equal(x, x_rec)
     assert_equal(type(x), type(x_rec))
     x = np.bool_(False)
     x_rec = self.encode_decode(x)
     assert_equal(x, x_rec)
     assert_equal(type(x), type(x_rec))
示例#12
0
    def get_results(self, conditions):
        
        
        self.conditions = dict(zip(conditions, range(len(conditions))))
        
        # Loads data for each subject
        # results is in the form (condition x subjects x runs x matrix)
        results = load_matrices(self.path, conditions)
        
        # Check if there are NaNs in the data
        nan_mask = np.isnan(results)
        for _ in range(len(results.shape) - 2):
            # For each condition/subject/run check if we have nan
            nan_mask = nan_mask.sum(axis=0)
        
        
            
        
        #pl.imshow(np.bool_(nan_mask), interpolation='nearest')
        #print np.nonzero(np.bool_(nan_mask)[0,:])
        # Clean NaNs
        results = results[:,:,:,~np.bool_(nan_mask)]
        
        # Reshaping because numpy masking flattens matrices        
        rows = np.sqrt(results.shape[-1])
        shape = list(results.shape[:-1])
        shape.append(int(rows))
        shape.append(-1)
        results = results.reshape(shape)
        
        # We apply z fisher to results
        zresults = z_fisher(results)
        zresults[np.isinf(zresults)] = 1
        
        self.results = zresults
        
        # Select mask to delete labels
        roi_mask = ~np.bool_(np.diagonal(nan_mask))

        # Get some information to store stuff
        self.store_details(roi_mask)   

        # Mean across runs
        zmean = zresults.mean(axis=2)
                
        new_shape = list(zmean.shape[-2:])
        new_shape.insert(0, -1)
        
        zreshaped = zmean.reshape(new_shape)
        
        upper_mask = np.ones_like(zreshaped[0])
        upper_mask[np.tril_indices(zreshaped[0].shape[0])] = 0
        upper_mask = np.bool_(upper_mask)
        
        # Returns the mask of the not available ROIs.
        self.nan_mask = nan_mask
        return self.nan_mask
    def test_index_bool( self ):
        a = np.arange(5)

        x = np.bool_(False)
        y = np.bool_(True)

        z = a[x:y]

        assert_equal( z, [0] )
示例#14
0
文件: test_sign.py 项目: BvanP/cvxpy
 def test_dmat_add(self):
     assert_equals(self.pos_vec + self.neg_vec, self.unknown_vec)
     result = Sign(np.bool_(True), self.arr)
     assert_equals(self.pos_vec + Sign.NEGATIVE, result)
     assert_equals(self.neg_vec + self.zero_vec, self.neg_vec)
     result = Sign(np.bool_(True), ~self.true_mat)
     assert_equals(self.neg_mat + Sign.NEGATIVE, result)
     assert_equals(self.pos_vec + self.pos_vec, self.pos_vec)
     assert_equals(self.unknown_mat + self.zero_mat, self.unknown_mat)
     assert_equals(Sign.UNKNOWN + self.pos_mat, Sign.UNKNOWN)
示例#15
0
 def test_astype(self):
     import numpy as np
     a = np.bool_(True).astype(np.float32)
     assert type(a) is np.float32
     assert a == 1.0
     a = np.bool_(True).astype('int32')
     assert type(a) is np.int32
     assert a == 1
     a = np.str_('123').astype('int32')
     assert type(a) is np.int32
     assert a == 123
 def test_mask_ratio(self):
     self.assertEqual(mask_ratio(True), 1)
     self.assertEqual(mask_ratio(np.bool_(True)), 1)
     self.assertEqual(mask_ratio(False), 0)
     self.assertEqual(mask_ratio(np.bool_(False)), 0)
     array = np.ma.array(range(10))
     self.assertEqual(mask_ratio(np.ma.getmaskarray(array)), 0)
     array[0] = np.ma.masked
     self.assertEqual(mask_ratio(np.ma.getmaskarray(array)), 0.1)
     invalid_array = np.ma.array(range(100), mask=[True]*100)
     self.assertEqual(mask_ratio(np.ma.getmaskarray(invalid_array)), 1)
示例#17
0
 def __init__(self, log_parameters=[], **kwargs):
     # Runs Individual.__init__ for default object initialization.
     super(Plankton, self).__init__(**kwargs)
     # Some parameters and initializations
     self.state_parameters = ['t', 'x', 'y', 'z', 'P', 'N_P']
     # ... history list
     self._attributes['history'] = dict()
     # ... log list
     n = self.size()
     extra = {key: [nan] * n for key in log_parameters}
     # Start history log
     self.append_history(extra=extra)
     # Assumes every individual is inside the model domain
     self.in_domain = bool_(self.x) | bool_(self.y)
示例#18
0
def construct_4d_adjacency_list(mask, numx=1, numy=1, numz=1, numt=1, nt=0):
    """
    Basically the prepare_adj function from regreg, but with less options.
    """
    
    regions = np.zeros(mask.shape)
    regions.shape = mask.shape
    reg_values = np.unique(regions)
    
    vmap = np.cumsum(mask).reshape(mask.shape)
    mask = np.bool_(mask.copy())
    vmap[~mask] = -1
    vmap -= 1 # sets vmap's values from 0 to mask.sum()-1
    
    adj = []
    nx, ny, nz = mask.shape
    
    for i, j, k, t in itertools.product(range(nx), range(ny),
                                        range(nz), range(nt)):
        
        if mask[i, j, k, t]:
            
            local_map = vmap[max((i-numx), 0):(i+numx+1),
                             max((j-numy), 0):(j+numy+1),
                             max((k-numz), 0):(k+numz+1),
                             max((t-numt), 0):(t+numt+1)]
            
            local_reg = regions[max((i-numx), 0):(i+numx+1),
                                max((j-numy), 0):(j+numy+1),
                                max((k-numz), 0):(k+numz+1),
                                max((t-numt), 0):(t+numt+1)]
            
            region = regions[i, j, k, t]
            ind = (local_map > -1) * (local_reg == region)
            ind = np.bool_(ind)
            nbrs = np.array(local_map[ind], dtype=np.int)
            adj.append(nbrs)
            
    
    for i, a in enumerate(adj):
        a[np.equal(a, i)] = -1
        
    num_ind = np.max([len(a) for a in adj])
    adjarray = -np.ones((len(adj), num_ind), dtype=np.int)
    
    for i in range(len(adj)):
        for j in range(len(adj[i])):
            adjarray[i,j] = adj[i][j]
            
    return adjarray
示例#19
0
def test_local_time_constraint_utc():
    time = Time('2001-02-03 04:05:06')
    subaru = Observer.at_site("Subaru")
    constraint = LocalTimeConstraint(min=dt.time(23, 50), max=dt.time(4, 8))
    is_constraint_met = constraint(subaru, None, times=time)
    assert is_constraint_met is np.bool_(True)

    constraint = LocalTimeConstraint(min=dt.time(0, 2), max=dt.time(4, 3))
    is_constraint_met = constraint(subaru, None, times=time)
    assert is_constraint_met is np.bool_(False)

    constraint = LocalTimeConstraint(min=dt.time(3, 8), max=dt.time(5, 35))
    is_constraint_met = constraint(subaru, None, times=time)
    assert is_constraint_met is np.bool_(True)
 def test_numpy_scalar_conversion_values(self):
     self.assertEqual(nd.as_py(nd.array(np.bool_(True))), True)
     self.assertEqual(nd.as_py(nd.array(np.bool_(False))), False)
     self.assertEqual(nd.as_py(nd.array(np.int8(100))), 100)
     self.assertEqual(nd.as_py(nd.array(np.int8(-100))), -100)
     self.assertEqual(nd.as_py(nd.array(np.int16(20000))), 20000)
     self.assertEqual(nd.as_py(nd.array(np.int16(-20000))), -20000)
     self.assertEqual(nd.as_py(nd.array(np.int32(1000000000))), 1000000000)
     self.assertEqual(nd.as_py(nd.array(np.int64(-1000000000000))),
                      -1000000000000)
     self.assertEqual(nd.as_py(nd.array(np.int64(1000000000000))),
                      1000000000000)
     self.assertEqual(nd.as_py(nd.array(np.int32(-1000000000))),
                      -1000000000)
     self.assertEqual(nd.as_py(nd.array(np.uint8(200))), 200)
     self.assertEqual(nd.as_py(nd.array(np.uint16(50000))), 50000)
     self.assertEqual(nd.as_py(nd.array(np.uint32(3000000000))), 3000000000)
     self.assertEqual(nd.as_py(nd.array(np.uint64(10000000000000000000))),
                      10000000000000000000)
     self.assertEqual(nd.as_py(nd.array(np.float32(2.5))), 2.5)
     self.assertEqual(nd.as_py(nd.array(np.float64(2.5))), 2.5)
     self.assertEqual(nd.as_py(nd.array(np.complex64(2.5-1j))), 2.5-1j)
     self.assertEqual(nd.as_py(nd.array(np.complex128(2.5-1j))), 2.5-1j)
     if np.__version__ >= '1.7':
         # Various date units
         self.assertEqual(nd.as_py(nd.array(np.datetime64('2000'))),
                          date(2000, 1, 1))
         self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12'))),
                          date(2000, 12, 1))
         self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13'))),
                          date(2000, 12, 13))
         # Various datetime units
         self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13T12Z'))),
                          datetime(2000, 12, 13, 12, 0))
         self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13T12:30Z'))),
                          datetime(2000, 12, 13, 12, 30))
         self.assertEqual(nd.as_py(nd.array(np.datetime64('1823-12-13T12:30Z'))),
                          datetime(1823, 12, 13, 12, 30))
         self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13T12:30:24Z'))),
                          datetime(2000, 12, 13, 12, 30, 24))
         self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13T12:30:24.123Z'))),
                          datetime(2000, 12, 13, 12, 30, 24, 123000))
         self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13T12:30:24.123456Z'))),
                          datetime(2000, 12, 13, 12, 30, 24, 123456))
         self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13T12:30:24.123456124Z'))),
                          datetime(2000, 12, 13, 12, 30, 24, 123456))
         self.assertEqual(str(nd.array(np.datetime64('2000-12-13T12:30:24.123456124Z'))),
                          '2000-12-13T12:30:24.1234561Z')
         self.assertEqual(str(nd.array(np.datetime64('1842-12-13T12:30:24.123456124Z'))),
                          '1842-12-13T12:30:24.1234561Z')
示例#21
0
 def __init__(self, endog, exog, censors):
     self.nobs = float(np.shape(exog)[0])
     self.endog = endog.reshape(self.nobs, 1)
     self.exog = exog.reshape(self.nobs, -1)
     self.censors = censors.reshape(self.nobs, 1)
     self.nvar = self.exog.shape[1]
     idx = np.lexsort((-self.censors[:, 0], self.endog[:, 0]))
     self.endog = self.endog[idx]
     self.exog = self.exog[idx]
     self.censors = self.censors[idx]
     self.censors[-1] = 1  # Sort in init, not in function
     self.uncens_nobs = np.sum(self.censors)
     self.uncens_endog = self.endog[np.bool_(self.censors), :].\
       reshape(-1, 1)
     self.uncens_exog = self.exog[np.bool_(self.censors.flatten()), :]
示例#22
0
def sign(constant):
    if isinstance(constant, numbers.Number):
        return Sign(np.bool_(constant < 0), np.bool_(constant > 0))
    elif isinstance(constant, cvxopt.spmatrix):
        # Convert to COO matrix.
        V = np.array(list(constant.V))
        I = list(constant.I)
        J = list(constant.J)
        # Check if entries > 0 for pos_mat, < 0 for neg_mat.
        neg_mat = sp.coo_matrix((V < 0,(I,J)), shape=constant.size, dtype='bool')
        pos_mat = sp.coo_matrix((V > 0,(I,J)), shape=constant.size, dtype='bool')
        return Sign(SparseBoolMat(neg_mat), SparseBoolMat(pos_mat))
    else:
        mat = INTERFACES[np.ndarray].const_to_matrix(constant)
        return Sign(mat < 0, mat > 0)
示例#23
0
    def test_operator_deprecation(self):
        array = np.array([True])
        generic = np.bool_(True)

        # Minus operator/subtract ufunc:
        self.assert_deprecated(operator.sub, args=(array, array))
        self.assert_deprecated(operator.sub, args=(generic, generic))
示例#24
0
    def test_array_flat_setitem(self):
        # Test indexing of array.flat object
        pyfunc = array_flat_setitem
        def check(arr, ind):
            arrty = typeof(arr)
            cr = self.ccache.compile(pyfunc, (arrty, typeof(ind), arrty.dtype))
            # Use np.copy() to keep the layout
            expected = np.copy(arr)
            got = np.copy(arr)
            pyfunc(expected, ind, 123)
            cr.entry_point(got, ind, 123)
            self.assertPreciseEqual(got, expected)

        arr = np.arange(24).reshape(4, 2, 3)
        for i in range(arr.size):
            check(arr, i)
        arr = arr.T
        for i in range(arr.size):
            check(arr, i)
        arr = arr[::2]
        for i in range(arr.size):
            check(arr, i)
        arr = np.array([42]).reshape(())
        for i in range(arr.size):
            check(arr, i)
        # Boolean array
        arr = np.bool_([1, 0, 0, 1])
        for i in range(arr.size):
            check(arr, i)
        arr = arr[::2]
        for i in range(arr.size):
            check(arr, i)
示例#25
0
    def test_bools(self):
        arr = np.array([True, False, True, True, True], dtype='O')
        result = lib.infer_dtype(arr)
        self.assertEqual(result, 'boolean')

        arr = np.array([np.bool_(True), np.bool_(False)], dtype='O')
        result = lib.infer_dtype(arr)
        self.assertEqual(result, 'boolean')

        arr = np.array([True, False, True, 'foo'], dtype='O')
        result = lib.infer_dtype(arr)
        self.assertEqual(result, 'mixed')

        arr = np.array([True, False, True], dtype=bool)
        result = lib.infer_dtype(arr)
        self.assertEqual(result, 'boolean')
示例#26
0
 def test_array_ndenumerate_2d(self):
     arr = np.arange(12).reshape(4, 3)
     arrty = typeof(arr)
     self.assertEqual(arrty.ndim, 2)
     self.assertEqual(arrty.layout, 'C')
     self.assertTrue(arr.flags.c_contiguous)
     # Test with C-contiguous array
     self.check_array_ndenumerate_sum(arr, arrty)
     # Test with Fortran-contiguous array
     arr = arr.transpose()
     self.assertFalse(arr.flags.c_contiguous)
     self.assertTrue(arr.flags.f_contiguous)
     arrty = typeof(arr)
     self.assertEqual(arrty.layout, 'F')
     self.check_array_ndenumerate_sum(arr, arrty)
     # Test with non-contiguous array
     arr = arr[::2]
     self.assertFalse(arr.flags.c_contiguous)
     self.assertFalse(arr.flags.f_contiguous)
     arrty = typeof(arr)
     self.assertEqual(arrty.layout, 'A')
     self.check_array_ndenumerate_sum(arr, arrty)
     # Boolean array
     arr = np.bool_([1, 0, 0, 1]).reshape((2, 2))
     self.check_array_ndenumerate_sum(arr, typeof(arr))
示例#27
0
    def check_nonzero(self, pyfunc):
        def fac(N):
            np.random.seed(42)
            arr = np.random.random(N)
            arr[arr < 0.3] = 0.0
            arr[arr > 0.7] = float('nan')
            return arr

        def check_arr(arr):
            cres = compile_isolated(pyfunc, (typeof(arr),))
            expected = pyfunc(arr)
            # NOTE: Numpy 1.9 returns readonly arrays for multidimensional
            # arrays.  Workaround this by copying the results.
            expected = [a.copy() for a in expected]
            self.assertPreciseEqual(cres.entry_point(arr), expected)

        arr = np.int16([1, 0, -1, 0])
        check_arr(arr)
        arr = np.bool_([1, 0, 1])
        check_arr(arr)

        arr = fac(24)
        check_arr(arr)
        check_arr(arr.reshape((3, 8)))
        check_arr(arr.reshape((3, 8)).T)
        check_arr(arr.reshape((3, 8))[::2])
        check_arr(arr.reshape((2, 3, 4)))
        check_arr(arr.reshape((2, 3, 4)).T)
        check_arr(arr.reshape((2, 3, 4))[::2])
        for v in (0.0, 1.5, float('nan')):
            arr = np.array([v]).reshape(())
            check_arr(arr)
示例#28
0
    def test_array_flat_getitem(self):
        # Test indexing of array.flat object
        pyfunc = array_flat_getitem
        def check(arr, ind):
            cr = self.ccache.compile(pyfunc, (typeof(arr), typeof(ind)))
            expected = pyfunc(arr, ind)
            self.assertEqual(cr.entry_point(arr, ind), expected)

        arr = np.arange(24).reshape(4, 2, 3)
        for i in range(arr.size):
            check(arr, i)
        arr = arr.T
        for i in range(arr.size):
            check(arr, i)
        arr = arr[::2]
        for i in range(arr.size):
            check(arr, i)
        arr = np.array([42]).reshape(())
        for i in range(arr.size):
            check(arr, i)
        # Boolean array
        arr = np.bool_([1, 0, 0, 1])
        for i in range(arr.size):
            check(arr, i)
        arr = arr[::2]
        for i in range(arr.size):
            check(arr, i)
示例#29
0
    def test_is_number(self):

        self.assertTrue(is_number(True))
        self.assertTrue(is_number(1))
        self.assertTrue(is_number(1.1))
        self.assertTrue(is_number(1 + 3j))
        self.assertTrue(is_number(np.bool(False)))
        self.assertTrue(is_number(np.int64(1)))
        self.assertTrue(is_number(np.float64(1.1)))
        self.assertTrue(is_number(np.complex128(1 + 3j)))
        self.assertTrue(is_number(np.nan))

        self.assertFalse(is_number(None))
        self.assertFalse(is_number('x'))
        self.assertFalse(is_number(datetime(2011, 1, 1)))
        self.assertFalse(is_number(np.datetime64('2011-01-01')))
        self.assertFalse(is_number(Timestamp('2011-01-01')))
        self.assertFalse(is_number(Timestamp('2011-01-01',
                                             tz='US/Eastern')))
        self.assertFalse(is_number(timedelta(1000)))
        self.assertFalse(is_number(Timedelta('1 days')))

        # questionable
        self.assertFalse(is_number(np.bool_(False)))
        self.assertTrue(is_number(np.timedelta64(1, 'D')))
示例#30
0
    def test_unary_minus_operator_deprecation(self):
        array = np.array([True])
        generic = np.bool_(True)

        # Unary minus/negative ufunc:
        self.assert_deprecated(operator.neg, args=(array,))
        self.assert_deprecated(operator.neg, args=(generic,))
 def test_base_json_conv(self):
     assert isinstance(base_json_conv(numpy.bool_(1)), bool) is True
     assert isinstance(base_json_conv(numpy.int64(1)), int) is True
     assert isinstance(base_json_conv(set([1])), list) is True
     assert isinstance(base_json_conv(Decimal('1.0')), float) is True
     assert isinstance(base_json_conv(uuid.uuid4()), str) is True
示例#32
0
    def test_np_where_3(self):
        pyfunc = np_where_3
        def fac(N):
            np.random.seed(42)
            arr = np.random.random(N)
            arr[arr < 0.3] = 0.0
            arr[arr > 0.7] = float('nan')
            return arr

        layouts = cycle(['C', 'F', 'A'])
        _types = [np.int32, np.int64, np.float32, np.float64, np.complex64,
                  np.complex128]

        np.random.seed(42)

        def check_arr(arr, layout=False):
            np.random.shuffle(_types)
            if layout != False:
                x = np.zeros_like(arr, dtype=_types[0], order=layout)
                y = np.zeros_like(arr, dtype=_types[1], order=layout)
                arr = arr.copy(order=layout)
            else:
                x = np.zeros_like(arr, dtype=_types[0], order=next(layouts))
                y = np.zeros_like(arr, dtype=_types[1], order=next(layouts))
            x.fill(4)
            y.fill(9)
            cres = compile_isolated(pyfunc, (typeof(arr), typeof(x), typeof(y)))
            expected = pyfunc(arr, x, y)
            got = cres.entry_point(arr, x, y)
            # Contiguity of result varies across Numpy versions, only
            # check contents. NumPy 1.11+ seems to stabilize.
            if numpy_version < (1, 11):
                self.assertEqual(got.dtype, expected.dtype)
                np.testing.assert_array_equal(got, expected)
            else:
                self.assertPreciseEqual(got, expected)

        def check_scal(scal):
            x = 4
            y = 5
            np.random.shuffle(_types)
            x = _types[0](4)
            y = _types[1](5)
            cres = compile_isolated(pyfunc, (typeof(scal), typeof(x), typeof(y)))
            expected = pyfunc(scal, x, y)
            got = cres.entry_point(scal, x, y)
            self.assertPreciseEqual(got, expected)

        arr = np.int16([1, 0, -1, 0])
        check_arr(arr)
        arr = np.bool_([1, 0, 1])
        check_arr(arr)

        arr = fac(24)
        check_arr(arr)
        check_arr(arr.reshape((3, 8)))
        check_arr(arr.reshape((3, 8)).T)
        check_arr(arr.reshape((3, 8))[::2])
        check_arr(arr.reshape((2, 3, 4)))
        check_arr(arr.reshape((2, 3, 4)).T)
        check_arr(arr.reshape((2, 3, 4))[::2])

        check_arr(arr.reshape((2, 3, 4)), layout='F')
        check_arr(arr.reshape((2, 3, 4)).T, layout='F')
        check_arr(arr.reshape((2, 3, 4))[::2], layout='F')

        for v in (0.0, 1.5, float('nan')):
            arr = np.array([v]).reshape(())
            check_arr(arr)

        for x in (0, 1, True, False, 2.5, 0j):
            check_scal(x)
示例#33
0
def test_table_typing_numpy():
    # Pulled from https://numpy.org/devdocs/user/basics.types.html

    # Numerics
    table = wandb.Table(columns=["A"], dtype=[NumberType])
    table.add_data(None)
    table.add_data(42)
    table.add_data(np.byte(1))
    table.add_data(np.short(42))
    table.add_data(np.ushort(42))
    table.add_data(np.intc(42))
    table.add_data(np.uintc(42))
    table.add_data(np.int_(42))
    table.add_data(np.uint(42))
    table.add_data(np.longlong(42))
    table.add_data(np.ulonglong(42))
    table.add_data(np.half(42))
    table.add_data(np.float16(42))
    table.add_data(np.single(42))
    table.add_data(np.double(42))
    table.add_data(np.longdouble(42))
    table.add_data(np.csingle(42))
    table.add_data(np.cdouble(42))
    table.add_data(np.clongdouble(42))
    table.add_data(np.int8(42))
    table.add_data(np.int16(42))
    table.add_data(np.int32(42))
    table.add_data(np.int64(42))
    table.add_data(np.uint8(42))
    table.add_data(np.uint16(42))
    table.add_data(np.uint32(42))
    table.add_data(np.uint64(42))
    table.add_data(np.intp(42))
    table.add_data(np.uintp(42))
    table.add_data(np.float32(42))
    table.add_data(np.float64(42))
    table.add_data(np.float_(42))
    table.add_data(np.complex64(42))
    table.add_data(np.complex128(42))
    table.add_data(np.complex_(42))

    # Booleans
    table = wandb.Table(columns=["A"], dtype=[BooleanType])
    table.add_data(None)
    table.add_data(True)
    table.add_data(False)
    table.add_data(np.bool_(True))

    # Array of Numerics
    table = wandb.Table(columns=["A"], dtype=[[NumberType]])
    table.add_data(None)
    table.add_data([42])
    table.add_data(np.array([1, 0], dtype=np.byte))
    table.add_data(np.array([42, 42], dtype=np.short))
    table.add_data(np.array([42, 42], dtype=np.ushort))
    table.add_data(np.array([42, 42], dtype=np.intc))
    table.add_data(np.array([42, 42], dtype=np.uintc))
    table.add_data(np.array([42, 42], dtype=np.int_))
    table.add_data(np.array([42, 42], dtype=np.uint))
    table.add_data(np.array([42, 42], dtype=np.longlong))
    table.add_data(np.array([42, 42], dtype=np.ulonglong))
    table.add_data(np.array([42, 42], dtype=np.half))
    table.add_data(np.array([42, 42], dtype=np.float16))
    table.add_data(np.array([42, 42], dtype=np.single))
    table.add_data(np.array([42, 42], dtype=np.double))
    table.add_data(np.array([42, 42], dtype=np.longdouble))
    table.add_data(np.array([42, 42], dtype=np.csingle))
    table.add_data(np.array([42, 42], dtype=np.cdouble))
    table.add_data(np.array([42, 42], dtype=np.clongdouble))
    table.add_data(np.array([42, 42], dtype=np.int8))
    table.add_data(np.array([42, 42], dtype=np.int16))
    table.add_data(np.array([42, 42], dtype=np.int32))
    table.add_data(np.array([42, 42], dtype=np.int64))
    table.add_data(np.array([42, 42], dtype=np.uint8))
    table.add_data(np.array([42, 42], dtype=np.uint16))
    table.add_data(np.array([42, 42], dtype=np.uint32))
    table.add_data(np.array([42, 42], dtype=np.uint64))
    table.add_data(np.array([42, 42], dtype=np.intp))
    table.add_data(np.array([42, 42], dtype=np.uintp))
    table.add_data(np.array([42, 42], dtype=np.float32))
    table.add_data(np.array([42, 42], dtype=np.float64))
    table.add_data(np.array([42, 42], dtype=np.float_))
    table.add_data(np.array([42, 42], dtype=np.complex64))
    table.add_data(np.array([42, 42], dtype=np.complex128))
    table.add_data(np.array([42, 42], dtype=np.complex_))

    # Array of Booleans
    table = wandb.Table(columns=["A"], dtype=[[BooleanType]])
    table.add_data(None)
    table.add_data([True])
    table.add_data([False])
    table.add_data(np.array([True, False], dtype=np.bool_))

    # Nested arrays
    table = wandb.Table(columns=["A"])
    table.add_data([[[[1, 2, 3]]]])
    table.add_data(np.array([[[[1, 2, 3]]]]))
示例#34
0
    def test_recursively_convert_to_json_serializable(self):
        D = ge.dataset.PandasDataset({
            'x': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
        })
        D.expect_column_values_to_be_in_set("x",
                                            set([1, 2, 3, 4, 5, 6, 7, 8, 9]),
                                            mostly=.8)

        part = ge.dataset.util.partition_data(D.x)
        D.expect_column_kl_divergence_to_be_less_than("x", part, .6)

        # Dumping this JSON object verifies that everything is serializable
        json.dumps(D.get_expectations_config(), indent=2)

        x = {
            'w': ["aaaa", "bbbb", 1.3, 5, 6, 7],
            'x': np.array([1, 2, 3]),
            'y': {
                'alpha': None,
                'beta': np.nan,
                'delta': np.inf,
                'gamma': -np.inf
            },
            'z': set([1, 2, 3, 4, 5]),
            'zz': (1, 2, 3),
            'zzz': [
                datetime.datetime(2017, 1, 1),
                datetime.date(2017, 5, 1),
            ],
            'np.bool': np.bool_([True, False, True]),
            'np.int_': np.int_([5, 3, 2]),
            'np.int8': np.int8([5, 3, 2]),
            'np.int16': np.int16([10, 6, 4]),
            'np.int32': np.int32([20, 12, 8]),
            'np.uint': np.uint([20, 5, 6]),
            'np.uint8': np.uint8([40, 10, 12]),
            'np.uint64': np.uint64([80, 20, 24]),
            'np.float_': np.float_([3.2, 5.6, 7.8]),
            'np.float32': np.float32([5.999999999, 5.6]),
            'np.float64': np.float64([5.9999999999999999999, 10.2]),
            'np.float128': np.float128([5.999999999998786324399999999, 20.4]),
            # 'np.complex64': np.complex64([10.9999999 + 4.9999999j, 11.2+7.3j]),
            # 'np.complex128': np.complex128([20.999999999978335216827+10.99999999j, 22.4+14.6j]),
            # 'np.complex256': np.complex256([40.99999999 + 20.99999999j, 44.8+29.2j]),
            'np.str': np.unicode_(["hello"]),
            'yyy': decimal.Decimal(123.456)
        }
        x = ge.dataset.util.recursively_convert_to_json_serializable(x)
        self.assertEqual(type(x['x']), list)

        self.assertEqual(type(x['np.bool'][0]), bool)
        self.assertEqual(type(x['np.int_'][0]), int)
        self.assertEqual(type(x['np.int8'][0]), int)
        self.assertEqual(type(x['np.int16'][0]), int)
        self.assertEqual(type(x['np.int32'][0]), int)

        # Integers in python 2.x can be of type int or of type long
        if sys.version_info.major >= 3:
            # Python 3.x
            self.assertTrue(isinstance(x['np.uint'][0], int))
            self.assertTrue(isinstance(x['np.uint8'][0], int))
            self.assertTrue(isinstance(x['np.uint64'][0], int))
        elif sys.version_info.major >= 2:
            # Python 2.x
            self.assertTrue(isinstance(x['np.uint'][0], (int, long)))
            self.assertTrue(isinstance(x['np.uint8'][0], (int, long)))
            self.assertTrue(isinstance(x['np.uint64'][0], (int, long)))

        self.assertEqual(type(x['np.float32'][0]), float)
        self.assertEqual(type(x['np.float64'][0]), float)
        self.assertEqual(type(x['np.float128'][0]), float)
        # self.assertEqual(type(x['np.complex64'][0]), complex)
        # self.assertEqual(type(x['np.complex128'][0]), complex)
        # self.assertEqual(type(x['np.complex256'][0]), complex)
        self.assertEqual(type(x['np.float_'][0]), float)

        # Make sure nothing is going wrong with precision rounding
        # self.assertAlmostEqual(x['np.complex128'][0].real, 20.999999999978335216827, places=sys.float_info.dig)
        self.assertAlmostEqual(x['np.float128'][0],
                               5.999999999998786324399999999,
                               places=sys.float_info.dig)

        # TypeError when non-serializable numpy object is in dataset.
        with self.assertRaises(TypeError):
            y = {'p': np.DataSource()}
            ge.dataset.util.recursively_convert_to_json_serializable(y)

        try:
            x = unicode("abcdefg")
            x = ge.dataset.util.recursively_convert_to_json_serializable(x)
            self.assertEqual(type(x), unicode)
        except NameError:
            pass
示例#35
0
# Default imports
import scipy.stats as stats
import pandas as pd
import numpy as np
from statsmodels.stats.weightstats import ztest
df = pd.read_csv('data/house_pricing.csv')
bol = np.bool_(dtype=bool)


# Enter Code Here
def t_statistic(data):
    #z_statistic, p_value = ztest(x1=data[data['Neighborhood'] == 'OldTown']['GrLivArea'], value=data['GrLivArea'].mean())
    z_statistic, p_value = stats.ttest_1samp(
        a=data[data['Neighborhood'] == 'OldTown']['GrLivArea'],
        popmean=data['GrLivArea'].mean())
    if z_statistic <= 0.05:
        bol = np.bool_(False)
    else:
        bol = np.bool_(True)
    return p_value, bol
示例#36
0
 def __init__(self):
     self.been_called = False
     self.ncalls = 0
     self.testres = [False, 'force accept', True, np.bool_(True),
                     np.bool_(False), [], {}, 0, 1]
示例#37
0
from typing import Any
import numpy as np

f8 = np.float64()
i8 = np.int64()
u8 = np.uint64()

f4 = np.float32()
i4 = np.int32()
u4 = np.uint32()

td = np.timedelta64(0, "D")
b_ = np.bool_()

b = bool()
f = float()
i = int()

AR_b: np.ndarray[Any, np.dtype[np.bool_]]
AR_m: np.ndarray[Any, np.dtype[np.timedelta64]]

# Time structures

reveal_type(td % td)  # E: numpy.timedelta64
reveal_type(AR_m % td)  # E: Any
reveal_type(td % AR_m)  # E: Any

reveal_type(divmod(td, td))  # E: Tuple[{int64}, numpy.timedelta64]
reveal_type(
    divmod(AR_m, td)
)  # E: Tuple[numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]], numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]]]
示例#38
0
    def test_numpy_scalars_ok(self, all_logical_operators):
        a = pd.array([True, False, None], dtype="boolean")
        op = getattr(a, all_logical_operators)

        tm.assert_extension_array_equal(op(True), op(np.bool_(True)))
        tm.assert_extension_array_equal(op(False), op(np.bool_(False)))
示例#39
0
        if arraysize != []:
            converter = converter.array_type(
                field, converter, arraysize, config)

        if not fixed:
            converter = converter.vararray_type(
                field, converter, arraysize, config)

    return converter


numpy_dtype_to_field_mapping = {
    np.float64().dtype.num    : 'double',
    np.float32().dtype.num    : 'float',
    np.bool_().dtype.num      : 'bit',
    np.uint8().dtype.num      : 'unsignedByte',
    np.int16().dtype.num      : 'short',
    np.int32().dtype.num      : 'int',
    np.int64().dtype.num      : 'long',
    np.complex64().dtype.num  : 'floatComplex',
    np.complex128().dtype.num : 'doubleComplex',
    np.unicode_().dtype.num   : 'unicodeChar'
}


numpy_dtype_to_field_mapping[np.bytes_().dtype.num] = 'char'


def _all_bytes(column):
    for x in column:
示例#40
0
class TestLogicalOps(BaseOpsUtil):
    def test_numpy_scalars_ok(self, all_logical_operators):
        a = pd.array([True, False, None], dtype="boolean")
        op = getattr(a, all_logical_operators)

        tm.assert_extension_array_equal(op(True), op(np.bool_(True)))
        tm.assert_extension_array_equal(op(False), op(np.bool_(False)))

    def get_op_from_name(self, op_name):
        short_opname = op_name.strip("_")
        short_opname = short_opname if "xor" in short_opname else short_opname + "_"
        try:
            op = getattr(operator, short_opname)
        except AttributeError:
            # Assume it is the reverse operator
            rop = getattr(operator, short_opname[1:])
            op = lambda x, y: rop(y, x)

        return op

    def test_empty_ok(self, all_logical_operators):
        a = pd.array([], dtype="boolean")
        op_name = all_logical_operators
        result = getattr(a, op_name)(True)
        tm.assert_extension_array_equal(a, result)

        result = getattr(a, op_name)(False)
        tm.assert_extension_array_equal(a, result)

        result = getattr(a, op_name)(pd.NA)
        tm.assert_extension_array_equal(a, result)

    def test_logical_length_mismatch_raises(self, all_logical_operators):
        op_name = all_logical_operators
        a = pd.array([True, False, None], dtype="boolean")
        msg = "Lengths must match to compare"

        with pytest.raises(ValueError, match=msg):
            getattr(a, op_name)([True, False])

        with pytest.raises(ValueError, match=msg):
            getattr(a, op_name)(np.array([True, False]))

        with pytest.raises(ValueError, match=msg):
            getattr(a, op_name)(pd.array([True, False], dtype="boolean"))

    def test_logical_nan_raises(self, all_logical_operators):
        op_name = all_logical_operators
        a = pd.array([True, False, None], dtype="boolean")
        msg = "Got float instead"

        with pytest.raises(TypeError, match=msg):
            getattr(a, op_name)(np.nan)

    @pytest.mark.parametrize("other", ["a", 1])
    def test_non_bool_or_na_other_raises(self, other, all_logical_operators):
        a = pd.array([True, False], dtype="boolean")
        with pytest.raises(TypeError, match=str(type(other).__name__)):
            getattr(a, all_logical_operators)(other)

    def test_kleene_or(self):
        # A clear test of behavior.
        a = pd.array([True] * 3 + [False] * 3 + [None] * 3, dtype="boolean")
        b = pd.array([True, False, None] * 3, dtype="boolean")
        result = a | b
        expected = pd.array(
            [True, True, True, True, False, None, True, None, None], dtype="boolean"
        )
        tm.assert_extension_array_equal(result, expected)

        result = b | a
        tm.assert_extension_array_equal(result, expected)

        # ensure we haven't mutated anything inplace
        tm.assert_extension_array_equal(
            a, pd.array([True] * 3 + [False] * 3 + [None] * 3, dtype="boolean")
        )
        tm.assert_extension_array_equal(
            b, pd.array([True, False, None] * 3, dtype="boolean")
        )

    @pytest.mark.parametrize(
        "other, expected",
        [
            (pd.NA, [True, None, None]),
            (True, [True, True, True]),
            (np.bool_(True), [True, True, True]),
            (False, [True, False, None]),
            (np.bool_(False), [True, False, None]),
        ],
    )
    def test_kleene_or_scalar(self, other, expected):
        # TODO: test True & False
        a = pd.array([True, False, None], dtype="boolean")
        result = a | other
        expected = pd.array(expected, dtype="boolean")
        tm.assert_extension_array_equal(result, expected)

        result = other | a
        tm.assert_extension_array_equal(result, expected)

        # ensure we haven't mutated anything inplace
        tm.assert_extension_array_equal(
            a, pd.array([True, False, None], dtype="boolean")
        )

    def test_kleene_and(self):
        # A clear test of behavior.
        a = pd.array([True] * 3 + [False] * 3 + [None] * 3, dtype="boolean")
        b = pd.array([True, False, None] * 3, dtype="boolean")
        result = a & b
        expected = pd.array(
            [True, False, None, False, False, False, None, False, None], dtype="boolean"
        )
        tm.assert_extension_array_equal(result, expected)

        result = b & a
        tm.assert_extension_array_equal(result, expected)

        # ensure we haven't mutated anything inplace
        tm.assert_extension_array_equal(
            a, pd.array([True] * 3 + [False] * 3 + [None] * 3, dtype="boolean")
        )
        tm.assert_extension_array_equal(
            b, pd.array([True, False, None] * 3, dtype="boolean")
        )

    @pytest.mark.parametrize(
        "other, expected",
        [
            (pd.NA, [None, False, None]),
            (True, [True, False, None]),
            (False, [False, False, False]),
            (np.bool_(True), [True, False, None]),
            (np.bool_(False), [False, False, False]),
        ],
    )
    def test_kleene_and_scalar(self, other, expected):
        a = pd.array([True, False, None], dtype="boolean")
        result = a & other
        expected = pd.array(expected, dtype="boolean")
        tm.assert_extension_array_equal(result, expected)

        result = other & a
        tm.assert_extension_array_equal(result, expected)

        # ensure we haven't mutated anything inplace
        tm.assert_extension_array_equal(
            a, pd.array([True, False, None], dtype="boolean")
        )

    def test_kleene_xor(self):
        a = pd.array([True] * 3 + [False] * 3 + [None] * 3, dtype="boolean")
        b = pd.array([True, False, None] * 3, dtype="boolean")
        result = a ^ b
        expected = pd.array(
            [False, True, None, True, False, None, None, None, None], dtype="boolean"
        )
        tm.assert_extension_array_equal(result, expected)

        result = b ^ a
        tm.assert_extension_array_equal(result, expected)

        # ensure we haven't mutated anything inplace
        tm.assert_extension_array_equal(
            a, pd.array([True] * 3 + [False] * 3 + [None] * 3, dtype="boolean")
        )
        tm.assert_extension_array_equal(
            b, pd.array([True, False, None] * 3, dtype="boolean")
        )

    @pytest.mark.parametrize(
        "other, expected",
        [
            (pd.NA, [None, None, None]),
            (True, [False, True, None]),
            (np.bool_(True), [False, True, None]),
            (np.bool_(False), [True, False, None]),
        ],
    )
    def test_kleene_xor_scalar(self, other, expected):
        a = pd.array([True, False, None], dtype="boolean")
        result = a ^ other
        expected = pd.array(expected, dtype="boolean")
        tm.assert_extension_array_equal(result, expected)

        result = other ^ a
        tm.assert_extension_array_equal(result, expected)

        # ensure we haven't mutated anything inplace
        tm.assert_extension_array_equal(
            a, pd.array([True, False, None], dtype="boolean")
        )

    @pytest.mark.parametrize("other", [True, False, pd.NA, [True, False, None] * 3])
    def test_no_masked_assumptions(self, other, all_logical_operators):
        # The logical operations should not assume that masked values are False!
        a = pd.arrays.BooleanArray(
            np.array([True, True, True, False, False, False, True, False, True]),
            np.array([False] * 6 + [True, True, True]),
        )
        b = pd.array([True] * 3 + [False] * 3 + [None] * 3, dtype="boolean")
        if isinstance(other, list):
            other = pd.array(other, dtype="boolean")

        result = getattr(a, all_logical_operators)(other)
        expected = getattr(b, all_logical_operators)(other)
        tm.assert_extension_array_equal(result, expected)

        if isinstance(other, BooleanArray):
            other._data[other._mask] = True
            a._data[a._mask] = False

            result = getattr(a, all_logical_operators)(other)
            expected = getattr(b, all_logical_operators)(other)
            tm.assert_extension_array_equal(result, expected)
示例#41
0
    def test_environment_stepper_component_with_large_impala_architecture(
            self):
        env_spec = dict(type="deepmind_lab",
                        level_id="seekavoid_arena_01",
                        observations=["RGB_INTERLEAVED", "INSTR"],
                        frameskip=4)
        dummy_env = Environment.from_spec(env_spec)
        state_space = dummy_env.state_space
        action_space = dummy_env.action_space
        actor_component = ActorComponent(
            # Preprocessor spec (only for image and prev-action channel).
            dict(
                type="dict-preprocessor-stack",
                preprocessors=dict(
                    ## The images from the env  are divided by 255.
                    #RGB_INTERLEAVED=[dict(type="divide", divisor=255)],
                    # The prev. action/reward from the env must be flattened/bumped-up-to-(1,).
                    previous_action=[
                        dict(type="reshape",
                             flatten=True,
                             flatten_categories=action_space.num_categories)
                    ],
                    previous_reward=[
                        dict(type="reshape", new_shape=(1, )),
                        dict(type="convert_type", to_dtype="float32")
                    ],
                )),
            # Policy spec.
            dict(network_spec=LargeIMPALANetwork(), action_space=action_space),
            # Exploration spec.
            Exploration(epsilon_spec=dict(decay_spec=dict(type="linear_decay",
                                                          from_=1.0,
                                                          to_=0.1,
                                                          start_timestep=0,
                                                          num_timesteps=100))))
        environment_stepper = EnvironmentStepper(
            environment_spec=env_spec,
            actor_component_spec=actor_component,
            state_space=state_space,
            reward_space="float32",
            internal_states_space=self.internal_states_space,
            num_steps=100,
            # Add both prev-action and -reward into the state sent through the network.
            add_previous_action_to_state=True,
            add_previous_reward_to_state=True,
            add_action_probs=True,
            action_probs_space=self.action_probs_space)

        test = ComponentTest(
            component=environment_stepper,
            action_space=action_space,
        )
        # Reset the stepper.
        test.test("reset")

        # Step n times through the Env and collect results.
        # 1st return value is the step-op (None), 2nd return value is the tuple of items (3 steps each), with each
        # step containing: Preprocessed state, actions, rewards, episode returns, terminals, (raw) next-states.
        time_start = time.perf_counter()
        steps = 10
        for _ in range(steps):
            out = test.test("step")
        time_total = time.perf_counter() - time_start
        print(
            "Done running {}x{} steps in Deepmind Lab env using IMPALA network in {}sec ({} actions/sec)."
            .format(steps, environment_stepper.num_steps, time_total,
                    environment_stepper.num_steps * steps / time_total))

        # Check types of outputs.
        self.assertTrue(isinstance(
            out, DataOpTuple))  # the step results as a tuple (see below)

        # Check types of single data.
        self.assertTrue(out[0]["INSTR"].dtype == np.object)
        self.assertTrue(out[0]["RGB_INTERLEAVED"].dtype == np.float32)
        self.assertTrue(out[0]["RGB_INTERLEAVED"].min() >=
                        0.0)  # make sure we have pixels / 255
        self.assertTrue(out[0]["RGB_INTERLEAVED"].max() <= 1.0)
        self.assertTrue(out[1].dtype == np.int32)  # actions
        self.assertTrue(out[2].dtype == np.float32)  # rewards
        self.assertTrue(out[3].dtype == np.float32)  # episode return
        self.assertTrue(out[4].dtype == np.bool_)  # next-state is terminal?
        self.assertTrue(out[5]["INSTR"].dtype ==
                        np.object)  # next state (raw, not preprocessed)
        self.assertTrue(out[5]["RGB_INTERLEAVED"].dtype ==
                        np.uint8)  # next state (raw, not preprocessed)
        self.assertTrue(
            out[5]["RGB_INTERLEAVED"].min() >= 0)  # make sure we have pixels
        self.assertTrue(out[5]["RGB_INTERLEAVED"].max() <= 255)
        # action probs (test whether sum to one).
        self.assertTrue(out[6].dtype == np.float32)
        self.assertTrue(out[6].min() >= 0.0)
        self.assertTrue(out[6].max() <= 1.0)
        recursive_assert_almost_equal(
            out[6].sum(axis=-1, keepdims=False),
            np.ones(shape=(environment_stepper.num_steps, )),
            decimals=4)
        # internal states (c- and h-state)
        self.assertTrue(out[7][0].dtype == np.float32)
        self.assertTrue(out[7][1].dtype == np.float32)
        self.assertTrue(out[7][0].shape == (environment_stepper.num_steps,
                                            256))
        self.assertTrue(out[7][1].shape == (environment_stepper.num_steps,
                                            256))

        # Check whether episode returns match single rewards (including terminal signals).
        episode_returns = 0.0
        for i in range(environment_stepper.num_steps):
            episode_returns += out[2][i]
            self.assertAlmostEqual(episode_returns, out[3][i])
            # Terminal: Reset for next step.
            if out[4][i] is np.bool_(True):
                episode_returns = 0.0

        test.terminate()
示例#42
0
def main(_):
    # CHECK-LABEL: TEST: abs int32[]
    # CHECK: mhlo.abs
    # CHECK-SAME: tensor<i32>
    print_ir(np.int32(0))(lax.abs)

    # CHECK-LABEL: TEST: add float32[] float32[]
    # CHECK: mhlo.add
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1), np.float32(2))(lax.add)

    # CHECK-LABEL: TEST: acos float32[]
    # CHECK: mhlo.atan2
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1))(lax.acos)

    # CHECK-LABEL: TEST: acosh float32[]
    # CHECK: chlo.acosh
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.acosh)

    # CHECK-LABEL: TEST: asin float32[]
    # CHECK: mhlo.atan2
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1))(lax.asin)

    # CHECK-LABEL: TEST: asinh float32[]
    # CHECK: chlo.asinh
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.asinh)

    # CHECK-LABEL: TEST: atan float32[]
    # CHECK: mhlo.atan2
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1))(lax.atan)

    # CHECK-LABEL: TEST: atanh float32[]
    # CHECK: chlo.atanh
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.atanh)

    # CHECK-LABEL: TEST: atan2 float64[] float64[]
    # CHECK: mhlo.atan2
    # CHECK-SAME: tensor<f64>
    print_ir(np.float64(1), np.float64(2))(lax.atan2)

    # CHECK-LABEL: TEST: bessel_i0e float32[]
    # CHECK: xla_fallback_bessel_i0e
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.bessel_i0e)

    # CHECK-LABEL: TEST: bessel_i1e float32[]
    # CHECK: xla_fallback_bessel_i1e
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.bessel_i1e)

    # CHECK-LABEL: TEST: betainc float32[] float32[] float32[]
    # CHECK: xla_fallback_regularized_incomplete_beta
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0), np.float32(0), np.float32(0))(lax.betainc)

    # CHECK-LABEL: TEST: bitcast_convert_type uint32[7]
    # CHECK: mhlo.bitcast_convert
    # CHECK-SAME: tensor<7xui32>
    # CHECK-SAME: tensor<7xf32>
    print_ir(np.empty((7, ), np.uint32))(partial(lax.bitcast_convert_type,
                                                 new_dtype=np.float32))

    # CHECK-LABEL: TEST: bitwise_and int32[] int32[]
    # CHECK: mhlo.and
    # CHECK-SAME: tensor<i32>
    print_ir(np.int32(1), np.int32(2))(lax.bitwise_and)

    # CHECK-LABEL: TEST: bitwise_and bool[] bool[]
    # CHECK: mhlo.and
    # CHECK-SAME: tensor<i1>
    print_ir(np.bool_(0), np.bool_(0))(lax.bitwise_and)

    # CHECK-LABEL: TEST: bitwise_or int32[] int32[]
    # CHECK: mhlo.or
    # CHECK-SAME: tensor<i32>
    print_ir(np.int32(1), np.int32(2))(lax.bitwise_or)

    # CHECK-LABEL: TEST: bitwise_or bool[] bool[]
    # CHECK: mhlo.or
    # CHECK-SAME: tensor<i1>
    print_ir(np.bool_(0), np.bool_(0))(lax.bitwise_or)

    # CHECK-LABEL: TEST: bitwise_xor int32[] int32[]
    # CHECK: mhlo.xor
    # CHECK-SAME: tensor<i32>
    print_ir(np.int32(1), np.int32(2))(lax.bitwise_xor)

    # CHECK-LABEL: TEST: bitwise_xor bool[] bool[]
    # CHECK: mhlo.xor
    # CHECK-SAME: tensor<i1>
    print_ir(np.bool_(0), np.bool_(0))(lax.bitwise_xor)

    # CHECK-LABEL: TEST: cbrt bfloat16[]
    # CHECK: mhlo.cbrt
    # CHECK-SAME: tensor<bf16>
    print_ir(jnp.bfloat16(0))(lax.cbrt)

    # CHECK-LABEL: TEST: clamp bfloat16[] bfloat16[] bfloat16[]
    # CHECK: mhlo.clamp
    # CHECK-SAME: tensor<bf16>
    print_ir(jnp.bfloat16(0), jnp.bfloat16(0), jnp.bfloat16(0))(lax.clamp)

    # CHECK-LABEL: TEST: ceil float16[7]
    # CHECK: mhlo.ceil
    # CHECK-SAME: tensor<7xf16>
    print_ir(np.empty((7, ), np.float16))(lax.ceil)

    # CHECK-LABEL: TEST: convert_element_type float16[7]
    # CHECK: mhlo.convert
    # CHECK-SAME: tensor<7xf16>
    # CHECK-SAME: tensor<7xf32>
    print_ir(np.empty((7, ), np.float16))(partial(lax.convert_element_type,
                                                  new_dtype=np.float32))

    # CHECK-LABEL: TEST: convert_element_type complex64[7]
    # CHECK: mhlo.real
    # CHECK-SAME: tensor<7xcomplex<f32>>
    # CHECK-SAME: tensor<7xf32>
    print_ir(np.empty((7, ), np.complex64))(partial(lax.convert_element_type,
                                                    new_dtype=np.float32))

    # CHECK-LABEL: TEST: convert_element_type float32[7]
    # CHECK: mhlo.compare
    # CHECK-SAME: tensor<7xf32>
    # CHECK-SAME: tensor<7xi1>
    print_ir(np.empty((7, ), np.float32))(partial(lax.convert_element_type,
                                                  new_dtype=np.bool_))

    # CHECK-LABEL: TEST: clz uint32[]
    # CHECK: mhlo.count_leading_zeros
    # CHECK-SAME: tensor<ui32>
    print_ir(np.uint32(0))(lax.clz)

    # CHECK-LABEL: TEST: conj complex64[]
    # CHECK-DAG: mhlo.real
    # CHECK-DAG: mhlo.imag
    # CHECK-DAG: mhlo.neg
    # CHECK-DAG: mhlo.complex
    # CHECK-SAME: tensor<complex<f32>>
    print_ir(np.complex64(0))(lax.conj)

    # CHECK-LABEL: TEST: cos float32[]
    # CHECK: mhlo.cos
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.cos)

    # CHECK-LABEL: TEST: cosh float32[]
    # CHECK: chlo.cosh
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.cosh)

    # CHECK-LABEL: TEST: digamma float32[]
    # CHECK: chlo.digamma
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.digamma)

    # CHECK-LABEL: TEST: div float32[] float32[]
    # CHECK: mhlo.div
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1), np.float32(2))(lax.div)

    # CHECK-LABEL: TEST: eq float32[] float32[]
    # CHECK: mhlo.compare
    # CHECK-SAME: compare_type = #mhlo<"comparison_type FLOAT">
    # CHECK-SAME: comparison_direction = #mhlo<"comparison_direction EQ">
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1), np.float32(2))(lax.eq)

    # CHECK-LABEL: TEST: eq complex128[] complex128[]
    # CHECK: mhlo.compare
    # CHECK-SAME: compare_type = #mhlo<"comparison_type FLOAT">
    # CHECK-SAME: comparison_direction = #mhlo<"comparison_direction EQ">
    # CHECK-SAME: tensor<complex<f64>>
    print_ir(np.complex128(1), np.complex128(2))(lax.eq)

    # CHECK-LABEL: TEST: eq int64[] int64[]
    # CHECK: mhlo.compare
    # CHECK-SAME: compare_type = #mhlo<"comparison_type SIGNED">
    # CHECK-SAME: comparison_direction = #mhlo<"comparison_direction EQ">
    # CHECK-SAME: tensor<i64>
    print_ir(np.int64(1), np.int64(2))(lax.eq)

    # CHECK-LABEL: TEST: eq uint16[] uint16[]
    # CHECK: mhlo.compare
    # CHECK-SAME: compare_type = #mhlo<"comparison_type UNSIGNED">
    # CHECK-SAME: comparison_direction = #mhlo<"comparison_direction EQ">
    # CHECK-SAME: tensor<ui16>
    print_ir(np.uint16(1), np.uint16(2))(lax.eq)

    # CHECK-LABEL: TEST: erf float32[]
    # CHECK: chlo.erf
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.erf)

    # CHECK-LABEL: TEST: erfc float32[]
    # CHECK: chlo.erfc
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.erfc)

    # CHECK-LABEL: TEST: erf_inv float32[]
    # CHECK: xla_fallback_erf_inv
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.erf_inv)

    # CHECK-LABEL: TEST: exp float16[]
    # CHECK: mhlo.exp
    # CHECK-SAME: tensor<f16>
    print_ir(np.float16(0))(lax.exp)

    # CHECK-LABEL: TEST: expm1 bfloat16[]
    # CHECK: mhlo.exponential_minus_one
    # CHECK-SAME: tensor<bf16>
    print_ir(jnp.bfloat16(0))(lax.expm1)

    # CHECK-LABEL: TEST: floor bfloat16[2,3]
    # CHECK: mhlo.floor
    # CHECK-SAME: tensor<2x3xbf16>
    print_ir(np.empty((2, 3), jnp.bfloat16))(lax.floor)

    # CHECK-LABEL: TEST: ge float32[] float32[]
    # CHECK: mhlo.compare
    # CHECK-SAME: compare_type = #mhlo<"comparison_type FLOAT">
    # CHECK-SAME: comparison_direction = #mhlo<"comparison_direction GE">
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1), np.float32(2))(lax.ge)

    # CHECK-LABEL: TEST: gt float32[] float32[]
    # CHECK: mhlo.compare
    # CHECK-SAME: compare_type = #mhlo<"comparison_type FLOAT">
    # CHECK-SAME: comparison_direction = #mhlo<"comparison_direction GT">
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1), np.float32(2))(lax.gt)

    # CHECK-LABEL: TEST: igamma float32[] float32[]
    # CHECK: xla_fallback_igamma
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0), np.float32(0))(lax.igamma)

    # CHECK-LABEL: TEST: igammac float32[] float32[]
    # CHECK: xla_fallback_igammac
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0), np.float32(0))(lax.igammac)

    # CHECK-LABEL: TEST: igamma_grad_a float32[] float32[]
    # CHECK: xla_fallback_igamma_grad_a
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0), np.float32(0))(lax.igamma_grad_a)

    # CHECK-LABEL: TEST: imag complex64[]
    # CHECK: mhlo.imag
    # CHECK-SAME: tensor<complex<f32>>
    print_ir(np.complex64(0))(lax.imag)

    # CHECK-LABEL: TEST: integer_pow float32[]
    # CHECK-DAG: mhlo.mul
    # CHECK-SAME: tensor<f32>
    @print_ir(np.float32(1))
    def integer_pow(x):
        return lax.integer_pow(x, 3)

    # CHECK-LABEL: TEST: is_finite float64[]
    # CHECK: mhlo.is_finite
    # CHECK-SAME: tensor<f64>
    print_ir(np.float64(0))(lax.is_finite)

    # CHECK-LABEL: TEST: le float32[] float32[]
    # CHECK: mhlo.compare
    # CHECK-SAME: compare_type = #mhlo<"comparison_type FLOAT">
    # CHECK-SAME: comparison_direction = #mhlo<"comparison_direction LE">
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1), np.float32(2))(lax.le)

    # CHECK-LABEL: TEST: lgamma float32[]
    # CHECK: chlo.lgamma
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.lgamma)

    # CHECK-LABEL: TEST: log float32[]
    # CHECK: mhlo.log
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.log)

    # CHECK-LABEL: TEST: log1p float32[]
    # CHECK: mhlo.log_plus_one
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.log1p)

    # CHECK-LABEL: TEST: lt float32[] float32[]
    # CHECK: mhlo.compare
    # CHECK-SAME: compare_type = #mhlo<"comparison_type FLOAT">
    # CHECK-SAME: comparison_direction = #mhlo<"comparison_direction LT">
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1), np.float32(2))(lax.lt)

    # CHECK-LABEL: TEST: max float32[] float32[]
    # CHECK: mhlo.max
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1), np.float32(2))(lax.max)

    # CHECK-LABEL: TEST: min float32[] float32[]
    # CHECK: mhlo.min
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1), np.float32(2))(lax.min)

    # CHECK-LABEL: TEST: mul float32[] float32[]
    # CHECK: mhlo.mul
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1), np.float32(2))(lax.mul)

    # CHECK-LABEL: TEST: ne float32[] float32[]
    # CHECK: mhlo.compare
    # CHECK-SAME: compare_type = #mhlo<"comparison_type FLOAT">
    # CHECK-SAME: comparison_direction = #mhlo<"comparison_direction NE">
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1), np.float32(2))(lax.ne)

    # CHECK-LABEL: TEST: neg int64[]
    # CHECK: mhlo.negate
    # CHECK-SAME: tensor<i64>
    print_ir(np.int64(0))(lax.neg)

    # CHECK-LABEL: TEST: nextafter float32[] float32[]
    # CHECK: chlo.next_after
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0), np.float32(0))(lax.nextafter)

    # CHECK-LABEL: TEST: bitwise_not int64[]
    # CHECK: mhlo.not
    # CHECK-SAME: tensor<i64>
    print_ir(np.int64(0))(lax.bitwise_not)

    # CHECK-LABEL: TEST: bitwise_not bool[]
    # CHECK: mhlo.not
    # CHECK-SAME: tensor<i1>
    print_ir(np.bool_(0))(lax.bitwise_not)

    # CHECK-LABEL: TEST: population_count uint32[]
    # CHECK: mhlo.popcnt
    # CHECK-SAME: tensor<ui32>
    print_ir(np.uint32(0))(lax.population_count)

    # CHECK-LABEL: TEST: pow float32[] float32[]
    # CHECK: mhlo.power
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1), np.float32(2))(lax.pow)

    # CHECK-LABEL: TEST: random_gamma_grad float32[] float32[]
    # CHECK: xla_fallback_random_gamma_grad
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0), np.float32(0))(lax.random_gamma_grad)

    # CHECK-LABEL: TEST: real complex128[]
    # CHECK: mhlo.real
    # CHECK-SAME: tensor<complex<f64>>
    print_ir(np.complex128(0))(lax.real)

    # CHECK-LABEL: TEST: reduce_precision bfloat16[]
    # CHECK: mhlo.reduce_precision
    # CHECK-SAME: tensor<bf16>
    print_ir(jnp.bfloat16(0))(partial(lax.reduce_precision,
                                      exponent_bits=2,
                                      mantissa_bits=2))

    # CHECK-LABEL: TEST: rem float32[] float32[]
    # CHECK: mhlo.rem
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1), np.float32(2))(lax.rem)

    # CHECK-LABEL: TEST: round float64[7,1]
    # CHECK: mhlo.round
    # CHECK-SAME: tensor<7x1xf64>
    print_ir(np.empty((7, 1), np.float64))(partial(
        lax.round, rounding_method=lax.RoundingMethod.AWAY_FROM_ZERO))

    # CHECK-LABEL: TEST: rsqrt complex64[]
    # CHECK: mhlo.rsqrt
    # CHECK-SAME: tensor<complex<f32>>
    print_ir(jnp.complex64(0))(lax.rsqrt)

    # CHECK-LABEL: TEST: shift_left uint32[] uint32[]
    # CHECK: mhlo.shift_left
    # CHECK-SAME: tensor<ui32>
    print_ir(np.uint32(0), np.uint32(0))(lax.shift_left)

    # CHECK-LABEL: TEST: shift_right_arithmetic uint8[] uint8[]
    # CHECK: mhlo.shift_right_arithmetic
    # CHECK-SAME: tensor<ui8>
    print_ir(np.uint8(0), np.uint8(0))(lax.shift_right_arithmetic)

    # CHECK-LABEL: TEST: shift_right_logical uint16[] uint16[]
    # CHECK: mhlo.shift_right_logical
    # CHECK-SAME: tensor<ui16>
    print_ir(np.uint16(0), np.uint16(0))(lax.shift_right_logical)

    # CHECK-LABEL: TEST: sign int64[]
    # CHECK: mhlo.sign
    # CHECK-SAME: tensor<i64>
    print_ir(np.int64(0))(lax.sign)

    # CHECK-LABEL: TEST: sign uint32[]
    # CHECK: mhlo.compare
    # CHECK-SAME: tensor<ui32>
    print_ir(np.uint32(0))(lax.sign)

    # CHECK-LABEL: TEST: sin float32[]
    # CHECK: mhlo.sin
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.sin)

    # CHECK-LABEL: TEST: sinh float32[]
    # CHECK: chlo.sinh
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.sinh)

    # CHECK-LABEL: TEST: sub float32[] float32[]
    # CHECK: mhlo.sub
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(1), np.float32(2))(lax.sub)

    # CHECK-LABEL: TEST: sqrt bfloat16[]
    # CHECK: mhlo.sqrt
    # CHECK-SAME: tensor<bf16>
    print_ir(jnp.bfloat16(0))(lax.sqrt)

    # CHECK-LABEL: TEST: tan float16[]
    # CHECK: chlo.tan
    # CHECK-SAME: tensor<f16>
    print_ir(np.float16(0))(lax.tan)

    # CHECK-LABEL: TEST: tanh float32[]
    # CHECK: mhlo.tanh
    # CHECK-SAME: tensor<f32>
    print_ir(np.float32(0))(lax.tanh)
示例#43
0
 def fkt(a):
     return np.bool_(a)
示例#44
0
 def test_numpy_bool_(self):
     self.assertReceivedEqualsSent(numpy.bool_(True), True)
     self.assertReceivedEqualsSent(numpy.bool_(False), False)
示例#45
0
class Board:
    """
    Board manages the game state
    It handles all transformations from one valid game state to another
    """
    costs = {
        "Road": {
            "Wood": 1,
            "Brick": 1,
            "Wheat": 0,
            "Rock": 0,
            "Sheep": 0
        },
        "Settlement": {
            "Wood": 1,
            "Brick": 1,
            "Wheat": 1,
            "Rock": 0,
            "Sheep": 1
        },
        "City": {
            "Wood": 0,
            "Brick": 0,
            "Wheat": 2,
            "Rock": 3,
            "Sheep": 0
        },
        "Development Card": {
            "Wood": 0,
            "Brick": 0,
            "Wheat": 1,
            "Rock": 1,
            "Sheep": 1
        }
    }
    # cols are junctions, rows are roads
    road_adjacency = pd.read_csv("road.csv", header=0, index_col=0)
    road_adjacency = road_adjacency.apply(lambda x: np.bool_(x))
    # cols are junctions, rows are tiles
    tile_adjacency = pd.read_csv("tile.csv", header=0, index_col=0)
    tile_adjacency = tile_adjacency.apply(lambda x: np.bool_(x))

    @staticmethod
    def s2r(location):
        x = Board.road_adjacency.iloc[:, location]
        return x

    @staticmethod
    def r2s(location):
        x = Board.road_adjacency.iloc[location, :]
        return x

    @staticmethod
    def r2r(location):
        s = Board.r2s(location)
        r = np.any(Board.road_adjacency.loc[:, s], 1)
        r[location] = False
        return r

    @staticmethod
    def s2s(location):
        r = Board.s2r(location)
        s = np.any(Board.road_adjacency.loc[r, :], 0)
        s[location] = False
        return s

    @staticmethod
    def s2t(location):
        return Board.tile_adjacency.iloc[:, location]

    @staticmethod
    def t2s(location):
        return Board.tile_adjacency.iloc[location, :]

    def __init__(self,
                 manager,
                 n_players,
                 border_setup="standard",
                 tile_setup="random"):
        """
        This sets up everything prior to initial settlement placement.
        Here's what it does:
            Assigns resources to each tile, including robbing the Desert
            Assigns numbers to the tiles based on the predefined order

            Generates the port arrangement

            Shuffles the Dev card deck

            Initializes all roads and cities to 0

            Sets the turn counter and player counters to 0

        """

        self.manager = manager

        resources = [
            "Brick", "Wheat", "Sheep", "Sheep", "Rock", "Brick", "Wood",
            "Sheep", "Rock", "Wheat", "Wheat", "Wood", "Rock", "Wheat", "Wood",
            "Sheep", "Brick", "Wood", "Desert"
        ]
        if tile_setup == "random":
            random.shuffle(resources)
        elif tile_setup == "basic":
            pass
        else:
            raise ValueError("Bad Tile Setup")
        number_order = [
            5, 6, 11, 5, 8, 10, 9, 2, 10, 12, 9, 8, 3, 4, 3, 4, 6, 11
        ]
        self.tiles = []
        for resource in resources:
            if resource == "Desert":
                self.tiles.append(Tile("Desert", 0))
            else:
                self.tiles.append(Tile(resource, number_order.pop(0)))

        self.border = Border(border_setup)

        self.roads = np.zeros(72, dtype=np.int8)
        self.settlements = np.zeros(54, dtype=np.int8)
        self.cities = np.zeros(54, dtype=np.int8)

        self.devdeck = [
            *["Knight"] * 14, *["Monopoly"] * 2, *["Road Building"] * 2,
            *["Victory"] * 5, *["Year of Plenty"] * 2
        ]
        random.shuffle(self.devdeck)

        # Using an extra player
        # to make players array practically start at 0
        self.players = [Player(self) for x in range(n_players + 1)]

        self.last_turn = 0
        self.turn_number = 0

    def check_road_length(self, player, location):
        """
        This is the most computer-science intensive part of the game
        It crawls through the road-adjacency graph recursively
        to find the longest road
        """
        def crawler(used, location, mode="distance"):
            joints = Board.road_adjacency.iloc[:, location]
            if sum(joints) != 2:
                print("Bad adjacency graph: road doesn't have 2 endpoints")
            options = np.zeros(72, dtype=np.int8)
            for i, joint in enumerate(joints):
                options += self.roads * \
                    Board.road_adjacency.iloc[joint, :] == player
            options = options * (1 - used)
            here = np.zeros(72, dtype=np.int8)
            if sum(options) > 0:
                paths = []
                for i, option in enumerate(options):
                    if option == 1:
                        here = np.zeros(72, dtype=np.int8)
                        here[location] = 1
                        paths.append(crawler(used + here, i))
                if mode == "distance":
                    return max(paths) + 1
                elif mode == "ends":
                    return np.sum(paths, 0)
                return False
            else:
                if mode == "distance":
                    return 1
                elif mode == "ends":
                    return here
                return False

        right_points = Board.road_adjacency.iloc[:, location]
        for i, h in enumerate(right_points):
            if h == 1:
                right_points[h] = 0
        left_endpoints = crawler([right_points], location, mode="ends")
        longest_road = 0
        for i, endpoint in enumerate(left_endpoints):
            if endpoint == 1:
                longest_road = max(longest_road, crawler([],
                                                         i,
                                                         mode="distance"))
        print("Found a long road:", longest_road)
        if longest_road >= 5:
            if self.players[player].road_length < longest_road:
                self.players[player].road_length = longest_road
                for other_player in self.players:
                    if other_player.road_length >= longest_road:
                        break
                else:
                    for other_player in self.players:
                        player.longest_road = False
                    self.players[player].longest_road = True

    def build_road(self, player, location, free=False):
        if self.roads[location] == 0 and self.players[player].roads > 0:
            next_settlements = Board.r2s(location)
            next_roads = Board.r2r(location)
            has_settlement = sum(self.settlements[next_settlements] == player)
            has_city = sum(self.cities[next_settlements] == player)
            has_road = sum(self.roads[next_roads] == player)
            if has_settlement + has_road + has_city > 0:
                if free or self.players[player].spend(Board.costs["Road"]):
                    self.roads[location] = player
                    self.players[player].roads -= 1
                    self.check_road_length(player, location)
                    return True
                print("No Resources!")
                return False
            print("No connection!")
            return False
        print("You're Blocked!")
        return False

    def build_settlement(self, player, location, initial=False, getnear=False):
        blocking_locations = Board.s2s(location)
        if sum((self.settlements + self.cities) * blocking_locations) == 0:
            if sum(self.roads[Board.s2r(location)] == player) > 0:
                if self.players[player].spend(Board.costs["Settlement"]):
                    self.settlements[location] = player
                    self.players[player].addvp()
                    self.players[player].settlements -= 1
                    self.players[player].addport(self.border.port(location))
                    return True
                else:
                    print("No Resources!")
            elif initial == True:
                self.settlements[location] = player
                self.players[player].settlements -= 1
                self.players[player].addvp()
                self.players[player].addport(self.border.port(location))
                if getnear:
                    for i, t in enumerate(Board.s2t(location)):
                        if t:
                            self.players[player].get(self.tiles[i].give())
                return True
            else:
                print("No Road")
        else:
            print("You're Blocked")
        return False

    def build_city(self, player, location):
        if self.settlements[
                location] == player and self.players[player].cities > 0:
            if self.players[player].spend(Board.costs["City"]):
                self.settlements[location] = 0
                self.players[player].cities -= 1
                self.players[player].settlements += 1
                self.cities[location] = player
                return True
            print("No Resources!")
            return False
        print("You need a settlement there first!")
        return False

    def buy_devcard(self, player):
        if len(self.devdeck) > 0:
            if self.players[player].spend(Board.costs["Development Card"]):
                self.players[player].get_devcard(self.devdeck.pop())
                return True
            print("No Resources!")
            return False
        print("You losers used all the dev cards!")
        return False

    def roll(self, fix=None):
        if self.last_turn == len(self.players):
            self.last_turn = 1
            self.turn_number += 1
        else:
            self.last_turn += 1
        result = random.randint(1, 6) + random.randint(1, 6)
        if fix is not None:
            result = fix
        if result != 7:
            for i, tile in enumerate(self.tiles):
                if tile.produce(result):
                    spots = Board.t2s(i)
                    settlements = self.settlements[spots]
                    cities = self.cities[spots]
                    for settlement in settlements:
                        if settlement != 0:
                            self.players[settlement].get(tile.give(1))
                    for city in cities:
                        if city != 0:
                            self.players[city].get(tile.give(2))
            return result
        else:
            return 7

    def rob(self, player1, player2, location):
        if self.tiles[location].robber == False:
            spots = Board.t2s(location)
            if sum(self.settlements[spots] == player2) > 0 or sum(
                    self.cities[spots] == player2) > 0:
                for tile in self.tiles:
                    tile.clearrobber()
                self.tiles[location].rob()
                self.players[player1].get(self.players[player2].take_random())
                return True
            print("That player isn't next to that tile!")
            return False
        print("The robber must move to a different tile!")
        return False

    def knight(self, player, special):
        if self.rob(player, special[0], special[1]):
            self.players[player].knight_count += 1

            # Check for largest army
            if self.players[player].knight_count > 2:
                for i, _ in enumerate(self.players):
                    if i != player:
                        if self.players[i].knight_count >= self.players[
                                player].knight_count:
                            break
                else:  # If the loop didn't break
                    for p in self.players:
                        if p.largest_army:
                            p.largest_army = False
                            p.victory_points -= 2
                    self.players[player].largest_army = True
                    self.players[player].addvp()
                    self.players[player].addvp()
            return True
        return False

    def monopoly(self, player, special):
        if type(special) == str and special in bank_statement.keys():
            for i, player in enumerate(self.players):
                receipt = bank_statement.copy()
                resource_number = player.resources[special]
                receipt[special] = resource_number
                self.players[player].get(self.players[i].spend(receipt))
            return True
        print("Monopoly takes a bank statement!")
        return False

    def year_of_plenty(self, player, special):
        if type(special) == dict and len(special) == 5:
            if sum(special.values()) == 2:
                self.players[player].get(special)
                return True
            print("That wasn't two resources!")
            return False
        print("That wasn't a receipt!")
        return False

    def road_building(self, player, special):
        if len(special) == 2 and type(special[0]) == int and type(
                special[1]) == int:
            if self.build_road(player, special[0],
                               free=True) and self.build_road(
                                   player, special[1], free=True):
                return True
        print("That isn't 2 locations!")
        return False

    def activate_devcard(self, player, card, turn_number, special):
        if self.players[player].can_flip_devcard(card, self.turn_number):
            if card == "Knight":
                if self.knight(player, special):
                    self.players[player].flip_devcard(card, turn_number)
                    return True
            elif card == "Victory":
                self.players[player].addvp()
                self.players[player].flip_devcard(card, turn_number)
                return True
            elif card == "Monopoly":
                if self.monopoly(player, special):
                    self.players[player].flip_devcard(card, turn_number)
                    return True
            elif card == "Road Building":
                if self.road_building(player, special):
                    self.players[player].flip_devcard(card, turn_number)
                    return True
                return False
            elif card == "Year of Plenty":
                if self.year_of_plenty(player, special):
                    self.players[player].flip_devcard(card, turn_number)
                    return True
            else:
                print("Not a valid card")
                return False
        print("You can't use that dev card!")
        return False
示例#46
0
 def test_numpy_scalar_conversion_values(self):
     self.assertEqual(nd.as_py(nd.array(np.bool_(True))), True)
     self.assertEqual(nd.as_py(nd.array(np.bool_(False))), False)
     self.assertEqual(nd.as_py(nd.array(np.int8(100))), 100)
     self.assertEqual(nd.as_py(nd.array(np.int8(-100))), -100)
     self.assertEqual(nd.as_py(nd.array(np.int16(20000))), 20000)
     self.assertEqual(nd.as_py(nd.array(np.int16(-20000))), -20000)
     self.assertEqual(nd.as_py(nd.array(np.int32(1000000000))), 1000000000)
     self.assertEqual(nd.as_py(nd.array(np.int64(-1000000000000))),
                      -1000000000000)
     self.assertEqual(nd.as_py(nd.array(np.int64(1000000000000))),
                      1000000000000)
     self.assertEqual(nd.as_py(nd.array(np.int32(-1000000000))),
                      -1000000000)
     self.assertEqual(nd.as_py(nd.array(np.uint8(200))), 200)
     self.assertEqual(nd.as_py(nd.array(np.uint16(50000))), 50000)
     self.assertEqual(nd.as_py(nd.array(np.uint32(3000000000))), 3000000000)
     self.assertEqual(nd.as_py(nd.array(np.uint64(10000000000000000000))),
                      10000000000000000000)
     self.assertEqual(nd.as_py(nd.array(np.float32(2.5))), 2.5)
     self.assertEqual(nd.as_py(nd.array(np.float64(2.5))), 2.5)
     self.assertEqual(nd.as_py(nd.array(np.complex64(2.5 - 1j))), 2.5 - 1j)
     self.assertEqual(nd.as_py(nd.array(np.complex128(2.5 - 1j))), 2.5 - 1j)
     if np.__version__ >= '1.7':
         # Various date units
         self.assertEqual(nd.as_py(nd.array(np.datetime64('2000'))),
                          date(2000, 1, 1))
         self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12'))),
                          date(2000, 12, 1))
         self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13'))),
                          date(2000, 12, 13))
         # Various datetime units
         self.assertEqual(
             nd.as_py(nd.array(np.datetime64('2000-12-13T12Z'))),
             datetime(2000, 12, 13, 12, 0))
         self.assertEqual(
             nd.as_py(nd.array(np.datetime64('2000-12-13T12:30Z'))),
             datetime(2000, 12, 13, 12, 30))
         self.assertEqual(
             nd.as_py(nd.array(np.datetime64('1823-12-13T12:30Z'))),
             datetime(1823, 12, 13, 12, 30))
         self.assertEqual(
             nd.as_py(nd.array(np.datetime64('2000-12-13T12:30:24Z'))),
             datetime(2000, 12, 13, 12, 30, 24))
         self.assertEqual(
             nd.as_py(nd.array(np.datetime64('2000-12-13T12:30:24.123Z'))),
             datetime(2000, 12, 13, 12, 30, 24, 123000))
         self.assertEqual(
             nd.as_py(nd.array(
                 np.datetime64('2000-12-13T12:30:24.123456Z'))),
             datetime(2000, 12, 13, 12, 30, 24, 123456))
         self.assertEqual(
             nd.as_py(
                 nd.array(np.datetime64('2000-12-13T12:30:24.123456124Z'))),
             datetime(2000, 12, 13, 12, 30, 24, 123456))
         self.assertEqual(
             str(nd.array(np.datetime64('2000-12-13T12:30:24.123456124Z'))),
             '2000-12-13T12:30:24.1234561Z')
         self.assertEqual(
             str(nd.array(np.datetime64('1842-12-13T12:30:24.123456124Z'))),
             '1842-12-13T12:30:24.1234561Z')
示例#47
0
文件: relay.py 项目: tomzhang/myia
 def convert_bool(self, v, t):
     """Convert the scalar to a TVM array."""
     return tvm.runtime.ndarray.array(np.bool_(v), self.context)
示例#48
0
 def test_scalar_array_bool(self):
     # Numpy bools can be used as boolean index (python ones as of yet not)
     a = np.array(1)
     assert_equal(a[np.bool_(True)], a[np.array(True)])
     assert_equal(a[np.bool_(False)], a[np.array(False)])
示例#49
0
 def setup(self):
     self.int64 = np.int64(1)
     self.integer_array = np.array([[1, 2], [-1, 2]])
     self.boolean_array = np.array([[True, False], [False, False]])
     self.bool_ = np.bool_(False)
     self.tuple = (slice(0, 4, 2), ..., 1)
示例#50
0
def stream_network(Z, flats, sills, interiorbasins, cellsize):

    #[Iobj,SILLS,IntBasin] = identifyflats(Z);
    Z = Z.data
    Z_ravel = np.ravel(Z)
    nrc = Z_ravel.shape[0]

    Iobj = flats
    SILLS = sills
    IntBasin = interiorbasins

    # Here we choose the distance transform from outside the lakes to the inside and take the locations as sills where the distance is maximum.
    DD = scipy.ndimage.distance_transform_edt(np.bitwise_not(IntBasin))
    MaxIntIX = [0, 0]  #added to prevent MaxIntIX does not exist errors
    IntBasin_labels = skimage.measure.label(IntBasin)
    for r in np.arange(1, np.max(IntBasin_labels)):
        PixelIdxList = np.argwhere(IntBasin_labels == r)
        ixm = np.argmax(DD[IntBasin_labels == r])
        MaxIntIX = PixelIdxList[ixm]

        Iobj[PixelIdxList[0][0], PixelIdxList[0][1]] = 0
        SILLS[PixelIdxList[0][0], PixelIdxList[0][1]] = 1
    ixm = MaxIntIX
    Iobj[ixm[0], ixm[1]] = 0
    SILLS[ixm[0], ixm[1]] = 1

    # establish the connectivity between sills and flats
    #dem = ZintoDB;
    whereSILLS = np.argwhere(SILLS)
    rows = []
    cols = []
    for rowcol in whereSILLS:
        [row, col] = rowcol
        rows = np.append(rows, row)
        cols = np.append(cols, col)

    IXsill = [rows, cols]
    rowadd = [-1, -1, 0, 1, 1, 1, 0, -1]
    coladd = [0, 1, 1, 1, 0, -1, -1, -1]
    PreSillPixel = [0]
    for r in np.arange(0, 8):
        rowp = rows + rowadd[r]
        colp = cols + coladd[r]

        ValidRowColPair1 = np.bitwise_and(rowp > 0, colp > 0)
        ValidRowColPair2 = np.bitwise_and(rowp < Z.shape[0], colp < Z.shape[1])
        ValidRowColPair = np.bitwise_and(ValidRowColPair1, ValidRowColPair2)
        whereValidRowColPair = np.where(ValidRowColPair)

        IXPreSill = [rowp[whereValidRowColPair], colp[whereValidRowColPair]]
        I1 = np.ravel_multi_index([
            np.int_(rows[whereValidRowColPair]),
            np.int_(cols[whereValidRowColPair])
        ], Z.shape)
        I2 = np.ravel_multi_index(
            [np.int_(IXPreSill[0]),
             np.int_(IXPreSill[1])], Z.shape)
        I3 = np.ravel_multi_index(
            [np.int_(IXPreSill[0]),
             np.int_(IXPreSill[1])], Z.shape)
        #PreSillPixelCondition = (np.argwhere(np.bitwise_and((Z_ravel[I1] ==
        #           Z_ravel[I2]),
        #          Iobj.ravel()[I3]))
        #   if np.count_nonzero(PreSillPixelCondition)>0:
        #       for i in  np.arange(0,len(PreSillPixelCondition)):
        #           PreSillPixelAddition = ([IXPreSill[0][PreSillPixelCondition[i]],IXPreSill[1][PreSillPixelCondition[i]]])
        #           PreSillPixel.append(PreSillPixelAddition)
        #   else:
        #       continue
    PreSillPixel.pop(0)

    Iobj = np.bitwise_not(Iobj)
    D = scipy.ndimage.distance_transform_edt(Iobj)
    masked = np.inf * np.ones(Z.shape, D.dtype)
    masked[Iobj] = 0
    D[np.bitwise_not(Iobj)] = np.inf
    D = ((skimage.morphology.reconstruction(
        seed=D + 1, mask=masked, method='erosion')) - D) * cellsize
    D = np.nan_to_num(D)

    D[Iobj] = 0
    #D = D**-1
    cost_field = map_image_to_costs(D**-1, PreSillPixel)
    D = _wdt_python(cost_field) + 1
    D[Iobj] = -np.inf

    #del PreSillPixel
    V = np.reshape(D.data, [1, D.shape[0] * D.shape[1]])
    if np.any(np.isnan(np.diff(D.ravel()))):
        IXSortedFlats = np.arange(0, len(Z_ravel))
        IXSortedFlats = IXSortedFlats[::-1]
    else:
        IXSortedFlats = np.argsort(D.ravel())
        IXSortedFlats = IXSortedFlats[::-1]
    #del D

    ndx = np.arange(np.uint32(0), np.uint32(nrc))
    ndx = ndx[IXSortedFlats]

    ndx = np.arange(np.uint32(0), np.uint32(nrc))
    ndx = ndx[IXSortedFlats]
    del IXSortedFlats

    ix = np.argsort(Z_ravel[ndx])
    ix = ix[::-1]
    ix = ndx[ix]
    del ndx

    # a fast solution that has quite much memory overhead...
    pp = np.zeros(Z_ravel.shape, dtype=np.int32)
    IX = np.arange(np.int32(0), np.int32(Z_ravel.shape))
    pp[ix] = IX
    pp = pp.reshape(Z.shape)

    # cardinal neighbors
    IXC1 = skimage.morphology.dilation(pp, skimage.morphology.selem.diamond(1))
    IXC1 = IXC1.ravel()
    xxx1 = IXC1
    IX = IXC1[ix]
    IXC1 = ix[IX]

    G1 = (Z_ravel[ix] - Z_ravel[IXC1]) / (cellsize)
    I4 = (np.argwhere(ix == IXC1)).ravel()

    I4 = list(I4)
    I4_test = np.zeros(G1.shape)
    I4_test[I4] = -np.inf
    G1 = G1 + I4_test
    #G1[ix == IXC1] = -np.inf;

    # diagonal neighbors
    kernel = np.array([[1, 0, 1], [0, 1, 0], [1, 0, 1]])
    IXC2 = skimage.morphology.dilation(pp, kernel)
    IXC2 = IXC2.ravel()
    xxx2 = IXC2
    IX = IXC2[ix]
    IXC2 = ix[IX]
    G2 = (Z_ravel[ix] - Z_ravel[IXC2]) / np.linalg.norm([cellsize, cellsize])

    # choose the steeper one
    #I  = np.bitwise_and(G1<=G2, xxx2[ix]>xxx1[ix]);
    I = dask.array.bitwise_and(dask.array.less_equal(G1, G2),
                               xxx2[ix] > xxx1[ix])
    ixc = IXC1
    ixc[I] = IXC2[I]

    I = ixc == ix
    ix = ix[np.bitwise_not(I)]
    ixc = ixc[np.bitwise_not(I)]

    # remove nans
    I = np.isnan(Z_ravel)
    ixc = ixc[~I[ix]]
    ix = ix[~I[ix]]

    ix = np.int_(ix[~np.isnan(ix)])
    ixc = np.int_(ixc[~np.isnan(ixc)])

    A = np.ones(Z.shape)
    for r in np.arange(0, len(ix)):
        [ixcx, ixcy] = np.unravel_index(ixc[r], Z.shape)
        [ixx, ixy] = np.unravel_index(ix[r], Z.shape)
        A[ixcx, ixcy] = A[ixx, ixy] + A[ixcx, ixcy]

    DBcounter = 0
    D = np.zeros(Z_ravel.shape[0], dtype=np.int32)
    outlets = np.zeros((len(ix), 1))
    for r in np.arange(len(ix) - 1, 1, -1):
        if D[ixc[r]] == 0:
            DBcounter = DBcounter + 1
            D[ixc[r]] = DBcounter
            outlets[DBcounter] = ixc[r]

        D[ix[r]] = D[ixc[r]]

    D = D.reshape(Z.shape)

    minarea = 1000
    W = np.bitwise_and(np.bool_(D > 0), np.bool_(A > minarea))
    Z = np.zeros(W.shape)
    [ixx, ixy] = np.unravel_index(ix, Z.shape)
    Z[ixx, ixy] = W[ixx, ixy]
    I = np.argwhere(Z[ixx, ixy])
    [ixcx, ixcy] = np.unravel_index(ix, Z.shape)
    Z[ixcx[I], ixcy[I]] = W[ixcx[I], ixcy[I]]
    W = Z * cellsize

    return W
np.timedelta64()
np.timedelta64(0)
np.timedelta64(0, "D")
np.timedelta64(0, ('ms', 3))
np.timedelta64(0, b"D")
np.timedelta64("3")
np.timedelta64(b"5")
np.timedelta64(np.timedelta64(2))
np.timedelta64(dt.timedelta(2))
np.timedelta64(None)
np.timedelta64(None, "D")

np.void(1)
np.void(np.int64(1))
np.void(True)
np.void(np.bool_(True))
np.void(b"test")
np.void(np.bytes_("test"))

# Protocols
i8 = np.int64()
u8 = np.uint64()
f8 = np.float64()
c16 = np.complex128()
b_ = np.bool_()
td = np.timedelta64()
U = np.str_("1")
S = np.bytes_("1")
AR = np.array(1, dtype=np.float64)

int(i8)
    elif typ is pd.CategoricalIndex:
        element = idx.categories[0]
        return pd.CategoricalIndex([element, element],
                                   categories=idx.categories,
                                   ordered=idx.ordered,
                                   name=idx.name)
    elif typ is pd.MultiIndex:
        levels = [_nonempty_index(i) for i in idx.levels]
        labels = [[0, 0] for i in idx.levels]
        return pd.MultiIndex(levels=levels, labels=labels, names=idx.names)
    raise TypeError("Don't know how to handle index of "
                    "type {0}".format(type(idx).__name__))


_simple_fake_mapping = {
    'b': np.bool_(True),
    'V': np.void(b' '),
    'M': np.datetime64('1970-01-01'),
    'm': np.timedelta64(1),
    'S': np.str_('foo'),
    'a': np.str_('foo'),
    'U': np.unicode_('foo'),
    'O': 'foo'
}


def _scalar_from_dtype(dtype):
    if dtype.kind in ('i', 'f', 'u'):
        return dtype.type(1)
    elif dtype.kind == 'c':
        return dtype.type(complex(1, 0))
 def test_numpybool_(self):
     nptrue = np.bool_(True)
     self.assertEqual(self.encoder.default(nptrue), True)
     self.assertIsInstance(self.encoder.default(nptrue), bool)
示例#54
0
def test_db2(name):
    if name == 'postgresql':
        pytest.importorskip('psycopg2')
        if os.environ.get('POSTGRES_DB'):  # gitlab-ci
            name = 'postgresql://*****:*****@postgres:5432/testase'
        else:
            name = os.environ.get('ASE_TEST_POSTGRES_URL')
            if name is None:
                return
    elif name == 'mysql':
        pytest.importorskip('pymysql')
        if os.environ.get('CI_PROJECT_DIR'):  # gitlab-ci
            name = 'mysql://*****:*****@mysql:3306/testase_mysql'
        else:
            name = os.environ.get('MYSQL_DB_URL')

        if name is None:
            return
    elif name == 'mariadb':
        pytest.importorskip('pymysql')
        if os.environ.get('CI_PROJECT_DIR'):  # gitlab-ci
            name = 'mariadb://*****:*****@mariadb:3306/testase_mysql'
        else:
            name = os.environ.get('MYSQL_DB_URL')

        if name is None:
            return

    c = connect(name)
    print(name, c)

    if 'postgres' in name or 'mysql' in name or 'mariadb' in name:
        c.delete([row.id for row in c.select()])

    id = c.reserve(abc=7)
    c.delete([d.id for d in c.select(abc=7)])
    id = c.reserve(abc=7)
    assert c[id].abc == 7

    a = c.get_atoms(id)
    c.write(Atoms())
    ch4 = molecule('CH4', calculator=EMT())
    ch4.constraints = [FixAtoms(indices=[1]), FixBondLength(0, 2)]
    f1 = ch4.get_forces()
    print(f1)

    c.delete([d.id for d in c.select(C=1)])
    chi = np.array([1 + 0.5j, 0.5])
    id = c.write(ch4, data={'1-butyne': 'bla-bla', 'chi': chi})

    row = c.get(id)
    print(row.data['1-butyne'], row.data.chi)
    assert (row.data.chi == chi).all(), (row.data.chi, chi)
    print(row)

    assert len(c.get_atoms(C=1).constraints) == 2

    f2 = c.get(C=1).forces
    assert abs(f2.sum(0)).max() < 1e-14
    f3 = c.get_atoms(C=1).get_forces()
    assert abs(f1 - f3).max() < 1e-14

    a = read(name, index='id={}'.format(id))[0]
    f4 = a.get_forces()
    assert abs(f1 - f4).max() < 1e-14

    with must_raise(ValueError):
        c.update(id, abc={'a': 42})

    c.update(id, grr='hmm')
    row = c.get(C=1)
    assert row.id == id
    assert (row.data.chi == chi).all()

    for row in c.select(include_data=False):
        assert len(row.data) == 0

    with must_raise(ValueError):
        c.write(ch4, foo=['bar', 2])  # not int, bool, float or str

    with must_raise(ValueError):
        c.write(Atoms(), pi='3.14')  # number as a string

    with must_raise(ValueError):
        c.write(Atoms(), fmax=0.0)  # reserved word

    with must_raise(ValueError):
        c.write(Atoms(), S=42)  # chemical symbol as key

    id = c.write(Atoms(),
                 b=np.bool_(True),
                 i=np.int64(42),
                 n=np.nan,
                 x=np.inf,
                 s='NaN2',
                 A=42)
    row = c[id]
    assert isinstance(row.b, bool)
    assert isinstance(row.i, int)
    assert np.isnan(row.n)
    assert np.isinf(row.x)

    # Make sure deleting a single key works:
    id = c.write(Atoms(), key=7)
    c.update(id, delete_keys=['key'])
    assert 'key' not in c[id]

    e = [row.get('energy') for row in c.select(sort='energy')]
    assert len(e) == 5 and abs(e[0] - 1.991) < 0.0005

    # Test the offset keyword
    ids = [row.get('id') for row in c.select()]
    offset = 2
    assert next(c.select(offset=offset)).id == ids[offset]
示例#55
0
def similarity_measure_euclidean(ds_tar, ds_src, results, p_value):

    print 'Computing Euclidean similarity...'
    #classifier = results['classifier']

    # Get classifier from results
    classifier = results['fclf']

    # Make prediction on training set, to understand data distribution
    prediction_src = classifier.predict(ds_src)
    #prediction_src = results['predictions_ds']
    true_predictions = np.array(prediction_src) == ds_src.targets
    example_dist = dict()

    # Extract feature selected from each dataset
    if isinstance(classifier, FeatureSelectionClassifier):
        f_selection = results['fclf'].mapper
        ds_tar = f_selection(ds_tar)
        ds_src = f_selection(ds_src)

    # Get no. of features
    df = ds_src.samples.shape[1]
    print 'dof ' + str(df)

    #Set a t distribution
    t_stu = scipy.stats.t(df)

    #Set the p-value and the threshold value to validate predictions
    m_value = t_stu.isf(p_value * 0.5)
    '''
    Get class distribution information: mean and covariance
    '''

    for label in np.unique(ds_src.targets):

        # Get examples correctly classified
        mask = ds_src.targets == label
        example_dist[label] = dict()
        true_ex = ds_src.samples[mask * true_predictions]

        # Get Mean and Covariance to draw the distribution
        mean_ = np.mean(true_ex, axis=0)
        example_dist[label]['mean'] = mean_
        std_ = np.std(true_ex, axis=0)
        example_dist[label]['std'] = std_.mean()
        example_dist[label]['threshold'] = euclidean(
            mean_, mean_ + m_value * std_.mean())

    # Get target predictions (unlabelled)
    prediction_target = results['predictions']

    # Test of data prediction
    euclidean_values = []
    normalized_values = []

    true_predictions = []

    for l, ex in zip(prediction_target, ds_tar.samples):

        # Keep mahalanobis distance between examples and class distribution
        dist = euclidean(example_dist[l]['mean'], ex)

        euclidean_values.append(dist)
        std_ = example_dist[l]['std']

        true_predictions.append(dist < example_dist[l]['threshold'])

        normalized_values.append(dist / np.sqrt(np.sum(np.power(std_, 2))))

    distances = dict()
    '''
    threshold = euclidean(example_dist['trained']['mean'], 
                          example_dist['untrained']['mean'])
    '''
    for c in np.unique(prediction_target):

        mean_ = example_dist[c]['mean']
        std_ = example_dist[c]['std']

        distances[c] = []

        for ex in ds_tar.samples:
            distances[c].append(euclidean(mean_, ex))

        distances[c] = np.array(distances[c])
        threshold = euclidean(mean_, mean_ + m_value * std_.mean())

        values = np.array(distances[c])
        print values.shape

        #true_predictions += (values < threshold)
    '''
    Squared Mahalanobis distance is similar to a chi square distribution with 
    degrees of freedom equal to the number of features.
    '''

    #mahalanobis_values = np.array(mahalanobis_values) ** 2

    print 'threshold ' + str(m_value)
    #print p_value

    #Mask true predictions

    p_values = 1 - 2 * t_stu.cdf(np.array(normalized_values))
    #print np.count_nonzero(p_values)
    '''
    Get some data
    '''
    full_data = np.array(
        zip(ds_tar.targets, prediction_target, euclidean_values, p_values))
    #print np.count_nonzero(np.float_(full_data.T[3]) == p_values)

    #true_data = full_data[true_predictions]

    return full_data, np.bool_(
        true_predictions), threshold, p_values, distances
示例#56
0
    def setData(self, index, value, role=Qt.DisplayRole):
        """Set the value to the index position depending on Qt::ItemDataRole and data type of the column

        Args:
            index (QtCore.QModelIndex): Index to define column and row.
            value (object): new value.
            role (Qt::ItemDataRole): Use this role to specify what you want to do.

        Raises:
            TypeError: If the value could not be converted to a known datatype.

        Returns:
            True if value is changed. Calls layoutChanged after update.
            False if value is not different from original value.

        """
        if not index.isValid() or not self.editable:
            return False

        if value != index.data(role):

            self.layoutAboutToBeChanged.emit()

            row = self._dataFrame.index[index.row()]
            col = self._dataFrame.columns[index.column()]
            #print 'before change: ', index.data().toUTC(), self._dataFrame.iloc[row][col]
            columnDtype = self._dataFrame[col].dtype

            if columnDtype == object:
                pass

            elif columnDtype in self._intDtypes:
                dtypeInfo = numpy.iinfo(columnDtype)
                if value < dtypeInfo.min:
                    value = dtypeInfo.min
                elif value > dtypeInfo.max:
                    value = dtypeInfo.max

            elif columnDtype in self._floatDtypes:
                value = numpy.float64(value).astype(columnDtype)

            elif columnDtype in self._boolDtypes:
                value = numpy.bool_(value)

            elif columnDtype in self._dateDtypes:
                # convert the given value to a compatible datetime object.
                # if the conversation could not be done, keep the original
                # value.
                if isinstance(value, QtCore.QDateTime):
                    value = value.toString(self.timestampFormat)
                try:
                    value = pandas.Timestamp(value)
                except Exception:
                    raise Exception(
                        "Can't convert '{0}' into a datetime".format(value))
                    # return False
            else:
                raise TypeError("try to set unhandled data type")

            self._dataFrame.set_value(row, col, value)

            #print 'after change: ', value, self._dataFrame.iloc[row][col]
            self.layoutChanged.emit()
            return True
        else:
            return False
示例#57
0
    scalar=(p.Scalar(), p.Scalar, -1),
    array=(p.Array(shape=(3, 2)), p.Array, -1),
    choice=(p.Choice([1, 2, 3]), p.Choice, 3),
    choice_weight=(p.Choice([1, 2, 3]).weights, p.Choice, 3),
)
def test_parameter_as_choice_tag(param: p.Parameter, cls: tp.Type[p.Parameter],
                                 arity: int) -> None:
    tag = p.BaseChoice.ChoiceTag.as_tag(param)
    assert tag.cls == cls
    assert tag.arity == arity


@testing.parametrized(
    true=(True, 0.0),
    false=(False, 1.0),
    np_true=(np.bool_(True), 0.0),
    np_false=(np.bool_(False), 1.0),
    pos=(0.7, 0.0),
    neg=(-0.7, 0.7),
    np_pos=(np.float(0.7), 0.0),
    np_neg=(np.float(-0.7), 0.7),
)
def test_float_penalty(value: tp.Any, expected: float) -> None:
    assert utils.float_penalty(value) == expected


def do_nothing(*args: tp.Any, **kwargs: tp.Any) -> int:
    print("my args", args, flush=True)
    print("my kwargs", kwargs, flush=True)
    if "sleep" in kwargs:
        print("Waiting", flush=True)
示例#58
0
    def process(self, df):
        if not df.size:
            return self.accumulator.identity()
        self._configure(df)
        dataset = df['dataset']
        df['is_lo_w'] = is_lo_w(dataset)
        df['is_lo_z'] = is_lo_z(dataset)
        df['is_lo_znunu'] = is_lo_znunu(dataset)
        df['is_lo_w_ewk'] = is_lo_w_ewk(dataset)
        df['is_lo_z_ewk'] = is_lo_z_ewk(dataset)
        df['is_lo_g'] = is_lo_g(dataset)
        df['is_nlo_z'] = is_nlo_z(dataset)
        df['is_nlo_w'] = is_nlo_w(dataset)
        df['has_lhe_v_pt'] = df['is_lo_w'] | df['is_lo_z'] | df[
            'is_nlo_z'] | df['is_nlo_w'] | df['is_lo_g'] | df[
                'is_lo_w_ewk'] | df['is_lo_z_ewk']
        df['is_data'] = is_data(dataset)

        gen_v_pt = None
        if df['is_lo_w'] or df['is_lo_z'] or df['is_nlo_z'] or df[
                'is_nlo_w'] or df['is_lo_z_ewk'] or df['is_lo_w_ewk']:
            gen = setup_gen_candidates(df)
            dressed = setup_dressed_gen_candidates(df)
            fill_gen_v_info(df, gen, dressed)
            gen_v_pt = df['gen_v_pt_combined']
        elif df['is_lo_g']:
            gen = setup_gen_candidates(df)
            all_gen_photons = gen[(gen.pdg == 22)]
            prompt_mask = (all_gen_photons.status
                           == 1) & (all_gen_photons.flag & 1 == 1)
            stat1_mask = (all_gen_photons.status == 1)
            gen_photons = all_gen_photons[prompt_mask |
                                          (~prompt_mask.any()) & stat1_mask]
            gen_photon = gen_photons[gen_photons.pt.argmax()]

            gen_v_pt = gen_photon.pt.max()

        # Generator-level leading dijet mass
        if df['has_lhe_v_pt']:
            genjets = setup_lhe_cleaned_genjets(df)
            digenjet = genjets[:, :2].distincts()
            df['mjj_gen'] = digenjet.mass.max()
            df['mjj_gen'] = np.where(df['mjj_gen'] > 0, df['mjj_gen'], 0)

        # Candidates
        # Already pre-filtered!
        # All leptons are at least loose
        # Check out setup_candidates for filtering details
        met_pt, met_phi, ak4, bjets, _, muons, electrons, taus, photons = setup_candidates(
            df, cfg)

        # Remove jets in accordance with the noise recipe
        if df['year'] == 2017:
            ak4 = ak4[(ak4.ptraw > 50) | (ak4.abseta < 2.65) |
                      (ak4.abseta > 3.139)]
            bjets = bjets[(bjets.ptraw > 50) | (bjets.abseta < 2.65) |
                          (bjets.abseta > 3.139)]

        # Filtering ak4 jets according to pileup ID
        ak4 = ak4[ak4.puid]

        # Muons
        df['is_tight_muon'] = muons.tightId \
                      & (muons.iso < cfg.MUON.CUTS.TIGHT.ISO) \
                      & (muons.pt>cfg.MUON.CUTS.TIGHT.PT) \
                      & (muons.abseta<cfg.MUON.CUTS.TIGHT.ETA)

        dimuons = muons.distincts()
        dimuon_charge = dimuons.i0['charge'] + dimuons.i1['charge']

        df['MT_mu'] = ((muons.counts == 1) *
                       mt(muons.pt, muons.phi, met_pt, met_phi)).max()

        # Electrons
        df['is_tight_electron'] = electrons.tightId \
                            & (electrons.pt > cfg.ELECTRON.CUTS.TIGHT.PT) \
                            & (electrons.absetasc < cfg.ELECTRON.CUTS.TIGHT.ETA)

        dielectrons = electrons.distincts()
        dielectron_charge = dielectrons.i0['charge'] + dielectrons.i1['charge']

        df['MT_el'] = ((electrons.counts == 1) *
                       mt(electrons.pt, electrons.phi, met_pt, met_phi)).max()

        # ak4
        leadak4_index = ak4.pt.argmax()

        elejet_pairs = ak4[:, :1].cross(electrons)
        df['dREleJet'] = np.hypot(
            elejet_pairs.i0.eta - elejet_pairs.i1.eta,
            dphi(elejet_pairs.i0.phi, elejet_pairs.i1.phi)).min()
        muonjet_pairs = ak4[:, :1].cross(muons)
        df['dRMuonJet'] = np.hypot(
            muonjet_pairs.i0.eta - muonjet_pairs.i1.eta,
            dphi(muonjet_pairs.i0.phi, muonjet_pairs.i1.phi)).min()

        # Recoil
        df['recoil_pt'], df['recoil_phi'] = recoil(met_pt, met_phi, electrons,
                                                   muons, photons)

        df["dPFCaloSR"] = (met_pt - df["CaloMET_pt"]) / met_pt
        df["dPFCaloCR"] = (met_pt - df["CaloMET_pt"]) / df["recoil_pt"]

        df["dPFTkSR"] = (met_pt - df["TkMET_pt"]) / met_pt

        df["minDPhiJetRecoil"] = min_dphi_jet_met(ak4,
                                                  df['recoil_phi'],
                                                  njet=4,
                                                  ptmin=30,
                                                  etamax=5.0)
        df["minDPhiJetMet"] = min_dphi_jet_met(ak4,
                                               met_phi,
                                               njet=4,
                                               ptmin=30,
                                               etamax=5.0)
        selection = processor.PackedSelection()

        # Triggers
        pass_all = np.ones(df.size) == 1
        selection.add('inclusive', pass_all)
        selection = trigger_selection(selection, df, cfg)

        selection.add('mu_pt_trig_safe', muons.pt.max() > 30)

        # Common selection
        selection.add('veto_ele', electrons.counts == 0)
        selection.add('veto_muo', muons.counts == 0)
        selection.add('veto_photon', photons.counts == 0)
        selection.add('veto_tau', taus.counts == 0)
        selection.add('at_least_one_tau', taus.counts > 0)
        selection.add('mindphijr',
                      df['minDPhiJetRecoil'] > cfg.SELECTION.SIGNAL.MINDPHIJR)
        selection.add('mindphijm',
                      df['minDPhiJetMet'] > cfg.SELECTION.SIGNAL.MINDPHIJR)

        # B jets are treated using veto weights
        # So accept them in MC, but reject in data
        if df['is_data']:
            selection.add('veto_b', bjets.counts == 0)
        else:
            selection.add('veto_b', pass_all)

        selection.add('dpfcalo_sr',
                      np.abs(df['dPFCaloSR']) < cfg.SELECTION.SIGNAL.DPFCALO)
        selection.add('dpfcalo_cr',
                      np.abs(df['dPFCaloCR']) < cfg.SELECTION.SIGNAL.DPFCALO)

        selection.add('recoil', df['recoil_pt'] > cfg.SELECTION.SIGNAL.RECOIL)
        selection.add('met_sr', met_pt > cfg.SELECTION.SIGNAL.RECOIL)

        # AK4 dijet
        diak4 = ak4[:, :2].distincts()
        leadak4_pt_eta = (diak4.i0.pt > cfg.SELECTION.SIGNAL.LEADAK4.PT) & (
            np.abs(diak4.i0.eta) < cfg.SELECTION.SIGNAL.LEADAK4.ETA)
        trailak4_pt_eta = (diak4.i1.pt > cfg.SELECTION.SIGNAL.TRAILAK4.PT) & (
            np.abs(diak4.i1.eta) < cfg.SELECTION.SIGNAL.TRAILAK4.ETA)
        hemisphere = (diak4.i0.eta * diak4.i1.eta < 0).any()
        has_track0 = np.abs(diak4.i0.eta) <= 2.5
        has_track1 = np.abs(diak4.i1.eta) <= 2.5

        leadak4_id = diak4.i0.tightId & (has_track0 * (
            (diak4.i0.chf > cfg.SELECTION.SIGNAL.LEADAK4.CHF) &
            (diak4.i0.nhf < cfg.SELECTION.SIGNAL.LEADAK4.NHF)) + ~has_track0)
        trailak4_id = has_track1 * (
            (diak4.i1.chf > cfg.SELECTION.SIGNAL.TRAILAK4.CHF) &
            (diak4.i1.nhf < cfg.SELECTION.SIGNAL.TRAILAK4.NHF)) + ~has_track1

        df['mjj'] = diak4.mass.max()
        df['dphijj'] = dphi(diak4.i0.phi.min(), diak4.i1.phi.max())
        df['detajj'] = np.abs(diak4.i0.eta - diak4.i1.eta).max()

        leading_jet_in_horn = ((diak4.i0.abseta < 3.2) &
                               (diak4.i0.abseta > 2.8)).any()
        trailing_jet_in_horn = ((diak4.i1.abseta < 3.2) &
                                (diak4.i1.abseta > 2.8)).any()

        selection.add('hornveto', (df['dPFTkSR'] < 0.8)
                      | ~(leading_jet_in_horn | trailing_jet_in_horn))

        if df['year'] == 2018:
            if df['is_data']:
                metphihem_mask = ~((met_phi > -1.8) & (met_phi < -0.6) &
                                   (df['run'] > 319077))
            else:
                metphihem_mask = pass_all
            selection.add("metphihemextveto", metphihem_mask)
            selection.add('no_el_in_hem',
                          electrons[electrons_in_hem(electrons)].counts == 0)
        else:
            selection.add("metphihemextveto", pass_all)
            selection.add('no_el_in_hem', pass_all)

        selection.add('two_jets', diak4.counts > 0)
        selection.add('leadak4_pt_eta', leadak4_pt_eta.any())
        selection.add('trailak4_pt_eta', trailak4_pt_eta.any())
        selection.add('hemisphere', hemisphere)
        selection.add('leadak4_id', leadak4_id.any())
        selection.add('trailak4_id', trailak4_id.any())
        selection.add('mjj',
                      df['mjj'] > cfg.SELECTION.SIGNAL.DIJET.SHAPE_BASED.MASS)
        selection.add(
            'dphijj',
            df['dphijj'] < cfg.SELECTION.SIGNAL.DIJET.SHAPE_BASED.DPHI)
        selection.add(
            'detajj',
            df['detajj'] > cfg.SELECTION.SIGNAL.DIJET.SHAPE_BASED.DETA)

        # Cleaning cuts for signal region

        # NEF cut: Only for endcap jets, require NEF < 0.7
        ak40_in_endcap = (diak4.i0.abseta > 2.5) & (diak4.i0.abseta < 3.0)
        ak41_in_endcap = (diak4.i1.abseta > 2.5) & (diak4.i1.abseta < 3.0)

        max_neEmEF_ak40 = (~ak40_in_endcap) | (diak4.i0.nef < 0.7)
        max_neEmEF_ak41 = (~ak41_in_endcap) | (diak4.i1.nef < 0.7)

        max_neEmEF = (max_neEmEF_ak40 & max_neEmEF_ak41).any()
        selection.add('max_neEmEF', max_neEmEF)

        vec_b = calculate_vecB(ak4, met_pt, met_phi)
        vec_dphi = calculate_vecDPhi(ak4, met_pt, met_phi, df['TkMET_phi'])

        no_jet_in_trk = (diak4.i0.abseta > 2.5).any() & (diak4.i1.abseta >
                                                         2.5).any()
        no_jet_in_hf = (diak4.i0.abseta < 3.0).any() & (diak4.i1.abseta <
                                                        3.0).any()

        at_least_one_jet_in_hf = (diak4.i0.abseta >
                                  3.0).any() | (diak4.i1.abseta > 3.0).any()
        at_least_one_jet_in_trk = (diak4.i0.abseta <
                                   2.5).any() | (diak4.i1.abseta < 2.5).any()

        # Categorized cleaning cuts
        eemitigation = ((no_jet_in_hf | at_least_one_jet_in_trk) &
                        (vec_dphi < 1.0)) | (
                            (no_jet_in_trk & at_least_one_jet_in_hf) &
                            (vec_b < 0.2))

        selection.add('eemitigation', eemitigation)

        # HF-HF veto in SR
        both_jets_in_hf = (diak4.i0.abseta > 3.0) & (diak4.i1.abseta > 3.0)
        selection.add('veto_hfhf', ~both_jets_in_hf.any())

        # Divide into three categories for trigger study
        if cfg.RUN.TRIGGER_STUDY:
            two_central_jets = (np.abs(diak4.i0.eta) <= 2.4) & (np.abs(
                diak4.i1.eta) <= 2.4)
            two_forward_jets = (np.abs(diak4.i0.eta) > 2.4) & (np.abs(
                diak4.i1.eta) > 2.4)
            one_jet_forward_one_jet_central = (~two_central_jets) & (
                ~two_forward_jets)
            selection.add('two_central_jets', two_central_jets.any())
            selection.add('two_forward_jets', two_forward_jets.any())
            selection.add('one_jet_forward_one_jet_central',
                          one_jet_forward_one_jet_central.any())

        # Dimuon CR
        leadmuon_index = muons.pt.argmax()
        selection.add('at_least_one_tight_mu', df['is_tight_muon'].any())
        selection.add('dimuon_mass', ((dimuons.mass > cfg.SELECTION.CONTROL.DOUBLEMU.MASS.MIN) \
                                    & (dimuons.mass < cfg.SELECTION.CONTROL.DOUBLEMU.MASS.MAX)).any())
        selection.add('dimuon_charge', (dimuon_charge == 0).any())
        selection.add('two_muons', muons.counts == 2)

        # Single muon CR
        selection.add('one_muon', muons.counts == 1)
        selection.add('mt_mu', df['MT_mu'] < cfg.SELECTION.CONTROL.SINGLEMU.MT)

        # Diele CR
        leadelectron_index = electrons.pt.argmax()

        selection.add('one_electron', electrons.counts == 1)
        selection.add('two_electrons', electrons.counts == 2)
        selection.add('at_least_one_tight_el', df['is_tight_electron'].any())


        selection.add('dielectron_mass', ((dielectrons.mass > cfg.SELECTION.CONTROL.DOUBLEEL.MASS.MIN)  \
                                        & (dielectrons.mass < cfg.SELECTION.CONTROL.DOUBLEEL.MASS.MAX)).any())
        selection.add('dielectron_charge', (dielectron_charge == 0).any())

        # Single Ele CR
        selection.add('met_el', met_pt > cfg.SELECTION.CONTROL.SINGLEEL.MET)
        selection.add('mt_el', df['MT_el'] < cfg.SELECTION.CONTROL.SINGLEEL.MT)

        # Photon CR
        leadphoton_index = photons.pt.argmax()

        df['is_tight_photon'] = photons.mediumId & photons.barrel

        selection.add('one_photon', photons.counts == 1)
        selection.add('at_least_one_tight_photon', df['is_tight_photon'].any())
        selection.add('photon_pt', photons.pt.max() > cfg.PHOTON.CUTS.TIGHT.PT)
        selection.add('photon_pt_trig',
                      photons.pt.max() > cfg.PHOTON.CUTS.TIGHT.PTTRIG)

        # Fill histograms
        output = self.accumulator.identity()

        # Gen
        if df['has_lhe_v_pt']:
            output['genvpt_check'].fill(vpt=gen_v_pt,
                                        type="Nano",
                                        dataset=dataset)

        if 'LHE_Njets' in df:
            output['lhe_njets'].fill(dataset=dataset,
                                     multiplicity=df['LHE_Njets'])
        if 'LHE_HT' in df:
            output['lhe_ht'].fill(dataset=dataset, ht=df['LHE_HT'])
        if 'LHE_HTIncoming' in df:
            output['lhe_htinc'].fill(dataset=dataset, ht=df['LHE_HTIncoming'])

        # Weights
        evaluator = evaluator_from_config(cfg)

        weights = processor.Weights(size=df.size, storeIndividual=True)
        if not df['is_data']:
            weights.add('gen', df['Generator_weight'])

            try:
                weights.add('prefire', df['PrefireWeight'])
            except KeyError:
                weights.add('prefire', np.ones(df.size))

            weights = candidate_weights(weights, df, evaluator, muons,
                                        electrons, photons, cfg)

            # B jet veto weights
            bsf_variations = btag_weights(bjets, cfg)
            weights.add("bveto", (1 - bsf_variations["central"]).prod())

            weights = pileup_weights(weights, df, evaluator, cfg)
            weights = ak4_em_frac_weights(weights, diak4, evaluator)
            if not (gen_v_pt is None):
                weights = theory_weights_vbf(weights, df, evaluator, gen_v_pt,
                                             df['mjj_gen'])

        # Save per-event values for synchronization
        if cfg.RUN.KINEMATICS.SAVE:
            for event in cfg.RUN.KINEMATICS.EVENTS:
                mask = df['event'] == event
                if not mask.any():
                    continue
                output['kinematics']['event'] += [event]
                output['kinematics']['met'] += [met_pt[mask]]
                output['kinematics']['met_phi'] += [met_phi[mask]]
                output['kinematics']['recoil'] += [df['recoil_pt'][mask]]
                output['kinematics']['recoil_phi'] += [df['recoil_phi'][mask]]

                output['kinematics']['ak4pt0'] += [ak4[leadak4_index][mask].pt]
                output['kinematics']['ak4eta0'] += [
                    ak4[leadak4_index][mask].eta
                ]
                output['kinematics']['leadbtag'] += [ak4.pt.max() < 0][mask]

                output['kinematics']['nLooseMu'] += [muons.counts[mask]]
                output['kinematics']['nTightMu'] += [
                    muons[df['is_tight_muon']].counts[mask]
                ]
                output['kinematics']['mupt0'] += [
                    muons[leadmuon_index][mask].pt
                ]
                output['kinematics']['mueta0'] += [
                    muons[leadmuon_index][mask].eta
                ]

                output['kinematics']['nLooseEl'] += [electrons.counts[mask]]
                output['kinematics']['nTightEl'] += [
                    electrons[df['is_tight_electron']].counts[mask]
                ]
                output['kinematics']['elpt0'] += [
                    electrons[leadelectron_index][mask].pt
                ]
                output['kinematics']['eleta0'] += [
                    electrons[leadelectron_index][mask].eta
                ]

                output['kinematics']['nLooseGam'] += [photons.counts[mask]]
                output['kinematics']['nTightGam'] += [
                    photons[df['is_tight_photon']].counts[mask]
                ]
                output['kinematics']['gpt0'] += [
                    photons[leadphoton_index][mask].pt
                ]
                output['kinematics']['geta0'] += [
                    photons[leadphoton_index][mask].eta
                ]

        # Sum of all weights to use for normalization
        # TODO: Deal with systematic variations
        output['nevents'][dataset] += df.size
        if not df['is_data']:
            output['sumw'][dataset] += df['genEventSumw']
            output['sumw2'][dataset] += df['genEventSumw2']
            output['sumw_pileup'][dataset] += weights._weights['pileup'].sum()

        regions = vbfhinv_regions(cfg)

        # Get veto weights (only for MC)
        if not df['is_data']:
            veto_weights = get_veto_weights(df, cfg, evaluator, electrons,
                                            muons, taus)

        for region, cuts in regions.items():
            exclude = [None]
            region_weights = copy.deepcopy(weights)

            if not df['is_data']:
                ### Trigger weights
                if re.match(r'cr_(\d+)e.*', region):
                    p_pass_data = 1 - (1 -
                                       evaluator["trigger_electron_eff_data"]
                                       (electrons.etasc, electrons.pt)).prod()
                    p_pass_mc = 1 - (1 - evaluator["trigger_electron_eff_mc"]
                                     (electrons.etasc, electrons.pt)).prod()
                    trigger_weight = p_pass_data / p_pass_mc
                    trigger_weight[np.isnan(trigger_weight)] = 1
                    region_weights.add('trigger', trigger_weight)
                elif re.match(r'cr_(\d+)m.*', region) or re.match(
                        'sr_.*', region):
                    met_trigger_sf(
                        region_weights,
                        diak4,
                        df,
                        apply_categorized=cfg.RUN.APPLY_CATEGORIZED_SF)
                elif re.match(r'cr_g.*', region):
                    photon_trigger_sf(region_weights, photons, df)

                # Veto weights
                if re.match('.*no_veto.*', region):
                    exclude = [
                        "muon_id_iso_tight", "muon_id_tight", "muon_iso_tight",
                        "muon_id_loose", "muon_iso_loose", "ele_reco",
                        "ele_id_tight", "ele_id_loose", "tau_id"
                    ]
                    region_weights.add(
                        "veto",
                        veto_weights.partial_weight(include=["nominal"]))

                # HEM-veto weights for signal region MC
                if re.match('^sr_vbf.*', region) and df['year'] == 2018:
                    # Events that lie in the HEM-veto region
                    events_to_weight_mask = (met_phi > -1.8) & (met_phi < -0.6)
                    # Weight is the "good lumi fraction" for 2018
                    weight = 21.1 / 59.7
                    hem_weight = np.where(events_to_weight_mask, weight, 1.0)

                    region_weights.add("hem_weight", hem_weight)

            # This is the default weight for this region
            rweight = region_weights.partial_weight(exclude=exclude)

            # Blinding
            if (self._blind and df['is_data'] and region.startswith('sr')):
                continue

            # Cutflow plot for signal and control regions
            if any(x in region for x in ["sr", "cr", "tr"]):
                output['cutflow_' + region][dataset]['all'] += df.size
                for icut, cutname in enumerate(cuts):
                    output['cutflow_' +
                           region][dataset][cutname] += selection.all(
                               *cuts[:icut + 1]).sum()

            mask = selection.all(*cuts)

            if cfg.RUN.SAVE.TREE:
                if region in ['cr_1e_vbf', 'cr_1m_vbf']:
                    output['tree_int64'][region][
                        "event"] += processor.column_accumulator(
                            df["event"][mask])
                    output['tree_float16'][region][
                        "gen_v_pt"] += processor.column_accumulator(
                            np.float16(gen_v_pt[mask]))
                    output['tree_float16'][region][
                        "gen_mjj"] += processor.column_accumulator(
                            np.float16(df['mjj_gen'][mask]))
                    output['tree_float16'][region][
                        "recoil_pt"] += processor.column_accumulator(
                            np.float16(df["recoil_pt"][mask]))
                    output['tree_float16'][region][
                        "recoil_phi"] += processor.column_accumulator(
                            np.float16(df["recoil_phi"][mask]))
                    output['tree_float16'][region][
                        "mjj"] += processor.column_accumulator(
                            np.float16(df["mjj"][mask]))

                    output['tree_float16'][region][
                        "leadak4_pt"] += processor.column_accumulator(
                            np.float16(diak4.i0.pt[mask]))
                    output['tree_float16'][region][
                        "leadak4_eta"] += processor.column_accumulator(
                            np.float16(diak4.i0.eta[mask]))
                    output['tree_float16'][region][
                        "leadak4_phi"] += processor.column_accumulator(
                            np.float16(diak4.i0.phi[mask]))

                    output['tree_float16'][region][
                        "trailak4_pt"] += processor.column_accumulator(
                            np.float16(diak4.i1.pt[mask]))
                    output['tree_float16'][region][
                        "trailak4_eta"] += processor.column_accumulator(
                            np.float16(diak4.i1.eta[mask]))
                    output['tree_float16'][region][
                        "trailak4_phi"] += processor.column_accumulator(
                            np.float16(diak4.i1.phi[mask]))

                    output['tree_float16'][region][
                        "minDPhiJetRecoil"] += processor.column_accumulator(
                            np.float16(df["minDPhiJetRecoil"][mask]))
                    if '_1e_' in region:
                        output['tree_float16'][region][
                            "leadlep_pt"] += processor.column_accumulator(
                                np.float16(electrons.pt.max()[mask]))
                        output['tree_float16'][region][
                            "leadlep_eta"] += processor.column_accumulator(
                                np.float16(electrons[
                                    electrons.pt.argmax()].eta.max()[mask]))
                        output['tree_float16'][region][
                            "leadlep_phi"] += processor.column_accumulator(
                                np.float16(electrons[
                                    electrons.pt.argmax()].phi.max()[mask]))
                    elif '_1m_' in region:
                        output['tree_float16'][region][
                            "leadlep_pt"] += processor.column_accumulator(
                                np.float16(muons.pt.max()[mask]))
                        output['tree_float16'][region][
                            "leadlep_eta"] += processor.column_accumulator(
                                np.float16(
                                    muons[muons.pt.argmax()].eta.max()[mask]))
                        output['tree_float16'][region][
                            "leadlep_phi"] += processor.column_accumulator(
                                np.float16(
                                    muons[muons.pt.argmax()].phi.max()[mask]))

                    for name, w in region_weights._weights.items():
                        output['tree_float16'][region][
                            f"weight_{name}"] += processor.column_accumulator(
                                np.float16(w[mask]))
                    output['tree_float16'][region][
                        f"weight_total"] += processor.column_accumulator(
                            np.float16(rweight[mask]))
                if region == 'inclusive':
                    output['tree_int64'][region][
                        "event"] += processor.column_accumulator(
                            df["event"][mask])
                    for name in selection.names:
                        output['tree_bool'][region][
                            name] += processor.column_accumulator(
                                np.bool_(selection.all(*[name])[mask]))
            # Save the event numbers of events passing this selection
            # Save the event numbers of events passing this selection
            if cfg.RUN.SAVE.PASSING:
                output['selected_events'][region] += list(df['event'][mask])

            # Multiplicities
            def fill_mult(name, candidates):
                output[name].fill(dataset=dataset,
                                  region=region,
                                  multiplicity=candidates[mask].counts,
                                  weight=rweight[mask])

            fill_mult('ak4_mult', ak4[ak4.pt > 30])
            fill_mult('bjet_mult', bjets)
            fill_mult('loose_ele_mult', electrons)
            fill_mult('tight_ele_mult', electrons[df['is_tight_electron']])
            fill_mult('loose_muo_mult', muons)
            fill_mult('tight_muo_mult', muons[df['is_tight_muon']])
            fill_mult('tau_mult', taus)
            fill_mult('photon_mult', photons)

            def ezfill(name, **kwargs):
                """Helper function to make filling easier."""
                output[name].fill(dataset=dataset, region=region, **kwargs)

            # Monitor weights
            for wname, wvalue in region_weights._weights.items():
                ezfill("weights", weight_type=wname, weight_value=wvalue[mask])
                ezfill("weights_wide",
                       weight_type=wname,
                       weight_value=wvalue[mask])

            # All ak4
            # This is a workaround to create a weight array of the right dimension
            w_alljets = weight_shape(ak4[mask].eta, rweight[mask])
            w_alljets_nopref = weight_shape(
                ak4[mask].eta,
                region_weights.partial_weight(exclude=exclude +
                                              ['prefire'])[mask])

            ezfill('ak4_eta', jeteta=ak4[mask].eta.flatten(), weight=w_alljets)
            ezfill('ak4_phi', jetphi=ak4[mask].phi.flatten(), weight=w_alljets)
            ezfill('ak4_pt', jetpt=ak4[mask].pt.flatten(), weight=w_alljets)

            ezfill('ak4_eta_nopref',
                   jeteta=ak4[mask].eta.flatten(),
                   weight=w_alljets_nopref)
            ezfill('ak4_phi_nopref',
                   jetphi=ak4[mask].phi.flatten(),
                   weight=w_alljets_nopref)
            ezfill('ak4_pt_nopref',
                   jetpt=ak4[mask].pt.flatten(),
                   weight=w_alljets_nopref)

            # Leading ak4
            w_diak4 = weight_shape(diak4.pt[mask], rweight[mask])
            ezfill('ak4_eta0',
                   jeteta=diak4.i0.eta[mask].flatten(),
                   weight=w_diak4)
            ezfill('ak4_phi0',
                   jetphi=diak4.i0.phi[mask].flatten(),
                   weight=w_diak4)
            ezfill('ak4_pt0',
                   jetpt=diak4.i0.pt[mask].flatten(),
                   weight=w_diak4)
            ezfill('ak4_ptraw0',
                   jetpt=diak4.i0.ptraw[mask].flatten(),
                   weight=w_diak4)
            ezfill('ak4_chf0',
                   frac=diak4.i0.chf[mask].flatten(),
                   weight=w_diak4)
            ezfill('ak4_nhf0',
                   frac=diak4.i0.nhf[mask].flatten(),
                   weight=w_diak4)
            ezfill('ak4_nconst0',
                   nconst=diak4.i0.nconst[mask].flatten(),
                   weight=w_diak4)

            # Trailing ak4
            ezfill('ak4_eta1',
                   jeteta=diak4.i1.eta[mask].flatten(),
                   weight=w_diak4)
            ezfill('ak4_phi1',
                   jetphi=diak4.i1.phi[mask].flatten(),
                   weight=w_diak4)
            ezfill('ak4_pt1',
                   jetpt=diak4.i1.pt[mask].flatten(),
                   weight=w_diak4)
            ezfill('ak4_ptraw1',
                   jetpt=diak4.i1.ptraw[mask].flatten(),
                   weight=w_diak4)
            ezfill('ak4_chf1',
                   frac=diak4.i1.chf[mask].flatten(),
                   weight=w_diak4)
            ezfill('ak4_nhf1',
                   frac=diak4.i1.nhf[mask].flatten(),
                   weight=w_diak4)
            ezfill('ak4_nconst1',
                   nconst=diak4.i1.nconst[mask].flatten(),
                   weight=w_diak4)

            # B tag discriminator
            btag = getattr(ak4, cfg.BTAG.ALGO)
            w_btag = weight_shape(btag[mask], rweight[mask])
            ezfill('ak4_btag', btag=btag[mask].flatten(), weight=w_btag)

            # MET
            ezfill('dpfcalo_cr',
                   dpfcalo=df["dPFCaloCR"][mask],
                   weight=rweight[mask])
            ezfill('dpfcalo_sr',
                   dpfcalo=df["dPFCaloSR"][mask],
                   weight=rweight[mask])
            ezfill('met', met=met_pt[mask], weight=rweight[mask])
            ezfill('met_phi', phi=met_phi[mask], weight=rweight[mask])
            ezfill('recoil',
                   recoil=df["recoil_pt"][mask],
                   weight=rweight[mask])
            ezfill('recoil_phi',
                   phi=df["recoil_phi"][mask],
                   weight=rweight[mask])
            ezfill('dphijm',
                   dphi=df["minDPhiJetMet"][mask],
                   weight=rweight[mask])
            ezfill('dphijr',
                   dphi=df["minDPhiJetRecoil"][mask],
                   weight=rweight[mask])

            ezfill('dphijj', dphi=df["dphijj"][mask], weight=rweight[mask])
            ezfill('detajj', deta=df["detajj"][mask], weight=rweight[mask])
            ezfill('mjj', mjj=df["mjj"][mask], weight=rweight[mask])

            # b-tag weight up and down variations
            if cfg.RUN.BTAG_STUDY:
                if not df['is_data']:
                    rw = region_weights.partial_weight(exclude=exclude +
                                                       ['bveto'])
                    ezfill('mjj_bveto_up',
                           mjj=df['mjj'][mask],
                           weight=(rw *
                                   (1 - bsf_variations['up']).prod())[mask])
                    ezfill('mjj_bveto_down',
                           mjj=df['mjj'][mask],
                           weight=(rw *
                                   (1 - bsf_variations['down']).prod())[mask])

            if gen_v_pt is not None:
                ezfill('gen_vpt',
                       vpt=gen_v_pt[mask],
                       weight=df['Generator_weight'][mask])
                ezfill('gen_mjj',
                       mjj=df['mjj_gen'][mask],
                       weight=df['Generator_weight'][mask])

            # Photon CR data-driven QCD estimate
            if df['is_data'] and re.match("cr_g.*", region) and re.match(
                    "(SinglePhoton|EGamma).*", dataset):
                w_imp = photon_impurity_weights(
                    photons[leadphoton_index].pt.max()[mask], df["year"])
                output['mjj'].fill(dataset=data_driven_qcd_dataset(dataset),
                                   region=region,
                                   mjj=df["mjj"][mask],
                                   weight=rweight[mask] * w_imp)
                output['recoil'].fill(dataset=data_driven_qcd_dataset(dataset),
                                      region=region,
                                      recoil=df["recoil_pt"][mask],
                                      weight=rweight[mask] * w_imp)

            # Uncertainty variations
            if df['is_lo_z'] or df['is_nlo_z'] or df['is_lo_z_ewk']:
                theory_uncs = [x for x in cfg.SF.keys() if x.startswith('unc')]
                for unc in theory_uncs:
                    reweight = evaluator[unc](gen_v_pt)
                    w = (region_weights.weight() * reweight)[mask]
                    ezfill('mjj_unc',
                           mjj=df['mjj'][mask],
                           uncertainty=unc,
                           weight=w)

            # Two dimensional
            ezfill('recoil_mjj',
                   recoil=df["recoil_pt"][mask],
                   mjj=df["mjj"][mask],
                   weight=rweight[mask])

            # Muons
            if '_1m_' in region or '_2m_' in region or 'no_veto' in region:
                w_allmu = weight_shape(muons.pt[mask], rweight[mask])
                ezfill('muon_pt', pt=muons.pt[mask].flatten(), weight=w_allmu)
                ezfill('muon_pt_abseta',
                       pt=muons.pt[mask].flatten(),
                       abseta=muons.eta[mask].flatten(),
                       weight=w_allmu)
                ezfill('muon_mt', mt=df['MT_mu'][mask], weight=rweight[mask])
                ezfill('muon_eta',
                       eta=muons.eta[mask].flatten(),
                       weight=w_allmu)
                ezfill('muon_phi',
                       phi=muons.phi[mask].flatten(),
                       weight=w_allmu)

            # Dimuon
            if '_2m_' in region:
                w_dimu = weight_shape(dimuons.pt[mask], rweight[mask])
                ezfill('muon_pt0',
                       pt=dimuons.i0.pt[mask].flatten(),
                       weight=w_dimu)
                ezfill('muon_pt1',
                       pt=dimuons.i1.pt[mask].flatten(),
                       weight=w_dimu)
                ezfill('muon_eta0',
                       eta=dimuons.i0.eta[mask].flatten(),
                       weight=w_dimu)
                ezfill('muon_eta1',
                       eta=dimuons.i1.eta[mask].flatten(),
                       weight=w_dimu)
                ezfill('muon_phi0',
                       phi=dimuons.i0.phi[mask].flatten(),
                       weight=w_dimu)
                ezfill('muon_phi1',
                       phi=dimuons.i1.phi[mask].flatten(),
                       weight=w_dimu)
                ezfill('dimuon_pt',
                       pt=dimuons.pt[mask].flatten(),
                       weight=w_dimu)
                ezfill('dimuon_eta',
                       eta=dimuons.eta[mask].flatten(),
                       weight=w_dimu)
                ezfill('dimuon_mass',
                       dilepton_mass=dimuons.mass[mask].flatten(),
                       weight=w_dimu)

            # Electrons
            if '_1e_' in region or '_2e_' in region or 'no_veto' in region:
                w_allel = weight_shape(electrons.pt[mask], rweight[mask])
                ezfill('electron_pt',
                       pt=electrons.pt[mask].flatten(),
                       weight=w_allel)
                ezfill('electron_pt_eta',
                       pt=electrons.pt[mask].flatten(),
                       eta=electrons.eta[mask].flatten(),
                       weight=w_allel)
                ezfill('electron_mt',
                       mt=df['MT_el'][mask],
                       weight=rweight[mask])
                ezfill('electron_eta',
                       eta=electrons.eta[mask].flatten(),
                       weight=w_allel)
                ezfill('electron_phi',
                       phi=electrons.phi[mask].flatten(),
                       weight=w_allel)

            # Dielectron
            if '_2e_' in region:
                w_diel = weight_shape(dielectrons.pt[mask], rweight[mask])
                ezfill('electron_pt0',
                       pt=dielectrons.i0.pt[mask].flatten(),
                       weight=w_diel)
                ezfill('electron_pt1',
                       pt=dielectrons.i1.pt[mask].flatten(),
                       weight=w_diel)
                ezfill('electron_eta0',
                       eta=dielectrons.i0.eta[mask].flatten(),
                       weight=w_diel)
                ezfill('electron_eta1',
                       eta=dielectrons.i1.eta[mask].flatten(),
                       weight=w_diel)
                ezfill('electron_phi0',
                       phi=dielectrons.i0.phi[mask].flatten(),
                       weight=w_diel)
                ezfill('electron_phi1',
                       phi=dielectrons.i1.phi[mask].flatten(),
                       weight=w_diel)
                ezfill('dielectron_pt',
                       pt=dielectrons.pt[mask].flatten(),
                       weight=w_diel)
                ezfill('dielectron_eta',
                       eta=dielectrons.eta[mask].flatten(),
                       weight=w_diel)
                ezfill('dielectron_mass',
                       dilepton_mass=dielectrons.mass[mask].flatten(),
                       weight=w_diel)

            # Photon
            if '_g_' in region:
                w_leading_photon = weight_shape(
                    photons[leadphoton_index].pt[mask], rweight[mask])
                ezfill('photon_pt0',
                       pt=photons[leadphoton_index].pt[mask].flatten(),
                       weight=w_leading_photon)
                ezfill('photon_eta0',
                       eta=photons[leadphoton_index].eta[mask].flatten(),
                       weight=w_leading_photon)
                ezfill('photon_phi0',
                       phi=photons[leadphoton_index].phi[mask].flatten(),
                       weight=w_leading_photon)
                ezfill('photon_pt0_recoil',
                       pt=photons[leadphoton_index].pt[mask].flatten(),
                       recoil=df['recoil_pt'][mask
                                              & (leadphoton_index.counts > 0)],
                       weight=w_leading_photon)
                ezfill('photon_eta_phi',
                       eta=photons[leadphoton_index].eta[mask].flatten(),
                       phi=photons[leadphoton_index].phi[mask].flatten(),
                       weight=w_leading_photon)

                # w_drphoton_jet = weight_shape(df['dRPhotonJet'][mask], rweight[mask])

            # Tau
            if 'no_veto' in region:
                w_all_taus = weight_shape(taus.pt[mask], rweight[mask])
                ezfill("tau_pt", pt=taus.pt[mask].flatten(), weight=w_all_taus)

            # PV
            ezfill('npv', nvtx=df['PV_npvs'][mask], weight=rweight[mask])
            ezfill('npvgood',
                   nvtx=df['PV_npvsGood'][mask],
                   weight=rweight[mask])

            ezfill('npv_nopu',
                   nvtx=df['PV_npvs'][mask],
                   weight=region_weights.partial_weight(exclude=exclude +
                                                        ['pileup'])[mask])
            ezfill('npvgood_nopu',
                   nvtx=df['PV_npvsGood'][mask],
                   weight=region_weights.partial_weight(exclude=exclude +
                                                        ['pileup'])[mask])

            ezfill('rho_all',
                   rho=df['fixedGridRhoFastjetAll'][mask],
                   weight=region_weights.partial_weight(exclude=exclude)[mask])
            ezfill('rho_central',
                   rho=df['fixedGridRhoFastjetCentral'][mask],
                   weight=region_weights.partial_weight(exclude=exclude)[mask])
            ezfill('rho_all_nopu',
                   rho=df['fixedGridRhoFastjetAll'][mask],
                   weight=region_weights.partial_weight(exclude=exclude +
                                                        ['pileup'])[mask])
            ezfill('rho_central_nopu',
                   rho=df['fixedGridRhoFastjetCentral'][mask],
                   weight=region_weights.partial_weight(exclude=exclude +
                                                        ['pileup'])[mask])
        return output
示例#59
0
@hash_object_dispatch.register((pd.DataFrame, pd.Series, pd.Index))
def hash_object_pandas(obj,
                       index=True,
                       encoding="utf8",
                       hash_key=None,
                       categorize=True):
    return pd.util.hash_pandas_object(obj,
                                      index=index,
                                      encoding=encoding,
                                      hash_key=hash_key,
                                      categorize=categorize)


_simple_fake_mapping = {
    "b": np.bool_(True),
    "V": np.void(b" "),
    "M": np.datetime64("1970-01-01"),
    "m": np.timedelta64(1),
    "S": np.str_("foo"),
    "a": np.str_("foo"),
    "U": np.unicode_("foo"),
    "O": "foo",
}


def _scalar_from_dtype(dtype):
    if dtype.kind in ("i", "f", "u"):
        return dtype.type(1)
    elif dtype.kind == "c":
        return dtype.type(complex(1, 0))
示例#60
0
# Program is part of MintPy                                #
# Copyright(c) 2016-2019, Zhang Yunjun                     #
# Author:  Zhang Yunjun                                    #
############################################################

import os
import re
import argparse
import datetime as dt
import h5py
import numpy as np
from mintpy.objects import timeseries, geometry, sensor
from mintpy.utils import readfile
from mintpy import info

BOOL_ZERO = np.bool_(0)
INT_ZERO = np.int16(0)
FLOAT_ZERO = np.float32(0.0)
CPX_ZERO = np.complex64(0.0)
compression = 'lzf'

################################################################
TEMPALTE = """
mintpy.save.hdfEos5         = auto   #[yes / no], auto for no, save timeseries to HDF-EOS5 format
mintpy.save.hdfEos5.update  = auto   #[yes / no], auto for no, put XXXXXXXX as endDate in output filename
mintpy.save.hdfEos5.subset  = auto   #[yes / no], auto for no, put subset range info   in output filename
"""

EXAMPLE = """example:
  save_hdfeos5.py geo_timeseries_ECMWF_ramp_demErr.h5 -c geo_temporalCoherence.h5 -m geo_maskTempCoh.h5 -g geo_geometryRadar.h5
"""