def mask_rho(self):
"""
Returns the mask for the cells
"""
if self._mask_rho is None:
mask_shape = tuple([n - 1 for n in self.x_vert.shape])
self._mask_rho = numpy.ones(mask_shape, dtype='d')
# If maskedarray is given for vertices, modify the mask such that
# non-existant grid points are masked. A cell requires all four
# verticies to be defined as a water point.
if isinstance(self.x_vert, numpy.ma.MaskedArray):
mask = (self.x_vert.mask[:-1, :-1] | self.x_vert.mask[1:, :-1] |
self.x_vert.mask[:-1, 1:] | self.x_vert.mask[1:, 1:])
self._mask_rho = numpy.asarray(
~(~numpy.bool_(self.mask_rho) | mask),
dtype='d'
)
if isinstance(self.y_vert, numpy.ma.MaskedArray):
mask = (self.y_vert.mask[:-1, :-1] | self.y_vert.mask[1:, :-1] |
self.y_vert.mask[:-1, 1:] | self.y_vert.mask[1:, 1:])
self._mask_rho = numpy.asarray(
~(~numpy.bool_(self.mask_rho) | mask),
dtype='d'
)
return self._mask_rho
def __init__(self, x, y):
assert np.ndim(x)==2 and np.ndim(y)==2 and np.shape(x)==np.shape(y), \
'x and y must be 2D arrays of the same size.'
if np.any(np.isnan(x)) or np.any(np.isnan(y)):
x = np.ma.masked_where( (isnan(x)) | (isnan(y)) , x)
y = np.ma.masked_where( (isnan(x)) | (isnan(y)) , y)
self.x_vert = x
self.y_vert = y
mask_shape = tuple([n-1 for n in self.x_vert.shape])
self.mask_rho = np.ones(mask_shape, dtype='d')
# If maskedarray is given for verticies, modify the mask such that
# non-existant grid points are masked. A cell requires all four
# verticies to be defined as a water point.
if isinstance(self.x_vert, np.ma.MaskedArray):
mask = (self.x_vert.mask[:-1,:-1] | self.x_vert.mask[1:,:-1] | \
self.x_vert.mask[:-1,1:] | self.x_vert.mask[1:,1:])
self.mask_rho = np.asarray(~(~np.bool_(self.mask_rho) | mask), dtype='d')
if isinstance(self.y_vert, np.ma.MaskedArray):
mask = (self.y_vert.mask[:-1,:-1] | self.y_vert.mask[1:,:-1] | \
self.y_vert.mask[:-1,1:] | self.y_vert.mask[1:,1:])
self.mask_rho = np.asarray(~(~np.bool_(self.mask_rho) | mask), dtype='d')
self._calculate_subgrids()
self._calculate_metrics()
def test00b_setBoolAttributes(self):
"""Checking setting Bool attributes (scalar, NumPy case)"""
self.array.attrs.pq = numpy.bool_(True)
self.array.attrs.qr = numpy.bool_(False)
self.array.attrs.rs = numpy.bool_(True)
# Check the results
if common.verbose:
print "pq -->", self.array.attrs.pq
print "qr -->", self.array.attrs.qr
print "rs -->", self.array.attrs.rs
if self.close:
if common.verbose:
print "(closing file version)"
self.fileh.close()
self.fileh = openFile(self.file, mode = "r+")
self.root = self.fileh.root
self.array = self.fileh.root.anarray
assert isinstance(self.root.anarray.attrs.pq, numpy.bool_)
assert isinstance(self.root.anarray.attrs.qr, numpy.bool_)
assert isinstance(self.root.anarray.attrs.rs, numpy.bool_)
assert self.root.anarray.attrs.pq == True
assert self.root.anarray.attrs.qr == False
assert self.root.anarray.attrs.rs == True
def test_numpy(self):
assert chash(np.bool_(True)) == chash(np.bool_(True))
assert chash(np.int8(1)) == chash(np.int8(1))
assert chash(np.int16(1))
assert chash(np.int32(1))
assert chash(np.int64(1))
assert chash(np.uint8(1))
assert chash(np.uint16(1))
assert chash(np.uint32(1))
assert chash(np.uint64(1))
assert chash(np.float32(1)) == chash(np.float32(1))
assert chash(np.float64(1)) == chash(np.float64(1))
assert chash(np.float128(1)) == chash(np.float128(1))
assert chash(np.complex64(1+1j)) == chash(np.complex64(1+1j))
assert chash(np.complex128(1+1j)) == chash(np.complex128(1+1j))
assert chash(np.complex256(1+1j)) == chash(np.complex256(1+1j))
assert chash(np.datetime64('2000-01-01')) == chash(np.datetime64('2000-01-01'))
assert chash(np.timedelta64(1,'W')) == chash(np.timedelta64(1,'W'))
self.assertRaises(ValueError, chash, np.object())
assert chash(np.array([[1, 2], [3, 4]])) == \
chash(np.array([[1, 2], [3, 4]]))
assert chash(np.array([[1, 2], [3, 4]])) != \
chash(np.array([[1, 2], [3, 4]]).T)
assert chash(np.array([1, 2, 3])) == chash(np.array([1, 2, 3]))
assert chash(np.array([1, 2, 3], dtype=np.int32)) != \
chash(np.array([1, 2, 3], dtype=np.int64))
def test_numpy_scalar_conversion_values(self):
self.assertEqual(nd.as_py(nd.array(np.bool_(True))), True)
self.assertEqual(nd.as_py(nd.array(np.bool_(False))), False)
self.assertEqual(nd.as_py(nd.array(np.int8(100))), 100)
self.assertEqual(nd.as_py(nd.array(np.int8(-100))), -100)
self.assertEqual(nd.as_py(nd.array(np.int16(20000))), 20000)
self.assertEqual(nd.as_py(nd.array(np.int16(-20000))), -20000)
self.assertEqual(nd.as_py(nd.array(np.int32(1000000000))), 1000000000)
self.assertEqual(nd.as_py(nd.array(np.int64(-1000000000000))),
-1000000000000)
self.assertEqual(nd.as_py(nd.array(np.int64(1000000000000))),
1000000000000)
self.assertEqual(nd.as_py(nd.array(np.int32(-1000000000))),
-1000000000)
self.assertEqual(nd.as_py(nd.array(np.uint8(200))), 200)
self.assertEqual(nd.as_py(nd.array(np.uint16(50000))), 50000)
self.assertEqual(nd.as_py(nd.array(np.uint32(3000000000))), 3000000000)
self.assertEqual(nd.as_py(nd.array(np.uint64(10000000000000000000))),
10000000000000000000)
self.assertEqual(nd.as_py(nd.array(np.float32(2.5))), 2.5)
self.assertEqual(nd.as_py(nd.array(np.float64(2.5))), 2.5)
self.assertEqual(nd.as_py(nd.array(np.complex64(2.5-1j))), 2.5-1j)
self.assertEqual(nd.as_py(nd.array(np.complex128(2.5-1j))), 2.5-1j)
if np.__version__ >= '1.7':
self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13'))),
date(2000, 12, 13))
def test_numpy_scalar_bool(self):
x = np.bool_(True)
x_rec = self.encode_decode(x)
assert x == x_rec and type(x) == type(x_rec)
x = np.bool_(False)
x_rec = self.encode_decode(x)
assert x == x_rec and type(x) == type(x_rec)
def eval(self, row, dataset):
result = np.bool_(self.value[0].eval(row, dataset))
for oper, val in self.operator_operands(self.value[1:]):
val = np.bool_(val.eval(row, dataset))
result = self.operation(oper, result, val)
return result
def test_normalize_json():
doc = {"foo": {numpy.bool_(True): "value"},
"what": numpy.bool_(False),
"this": numpy.PINF}
normalized_doc = normalize_json(doc)
assert isinstance(normalized_doc['what'], bool)
assert isinstance(list(normalized_doc['foo'].keys())[0], bool)
assert normalized_doc['this'] == "Infinity"
def test_scalar_bool(self):
x = np.bool_(1)
x_rec = self.encode_decode(x)
tm.assert_almost_equal(x, x_rec)
x = np.bool_(0)
x_rec = self.encode_decode(x)
tm.assert_almost_equal(x, x_rec)
def test_numpy_bool():
import numpy as np
convert, noconvert = m.bool_passthrough, m.bool_passthrough_noconvert
# np.bool_ is not considered implicit
assert convert(np.bool_(True)) is True
assert convert(np.bool_(False)) is False
assert noconvert(np.bool_(True)) is True
assert noconvert(np.bool_(False)) is False
def test_numpy_scalar_bool(self):
x = np.bool_(True)
x_rec = self.encode_decode(x)
assert_equal(x, x_rec)
assert_equal(type(x), type(x_rec))
x = np.bool_(False)
x_rec = self.encode_decode(x)
assert_equal(x, x_rec)
assert_equal(type(x), type(x_rec))
def get_results(self, conditions):
self.conditions = dict(zip(conditions, range(len(conditions))))
# Loads data for each subject
# results is in the form (condition x subjects x runs x matrix)
results = load_matrices(self.path, conditions)
# Check if there are NaNs in the data
nan_mask = np.isnan(results)
for _ in range(len(results.shape) - 2):
# For each condition/subject/run check if we have nan
nan_mask = nan_mask.sum(axis=0)
#pl.imshow(np.bool_(nan_mask), interpolation='nearest')
#print np.nonzero(np.bool_(nan_mask)[0,:])
# Clean NaNs
results = results[:,:,:,~np.bool_(nan_mask)]
# Reshaping because numpy masking flattens matrices
rows = np.sqrt(results.shape[-1])
shape = list(results.shape[:-1])
shape.append(int(rows))
shape.append(-1)
results = results.reshape(shape)
# We apply z fisher to results
zresults = z_fisher(results)
zresults[np.isinf(zresults)] = 1
self.results = zresults
# Select mask to delete labels
roi_mask = ~np.bool_(np.diagonal(nan_mask))
# Get some information to store stuff
self.store_details(roi_mask)
# Mean across runs
zmean = zresults.mean(axis=2)
new_shape = list(zmean.shape[-2:])
new_shape.insert(0, -1)
zreshaped = zmean.reshape(new_shape)
upper_mask = np.ones_like(zreshaped[0])
upper_mask[np.tril_indices(zreshaped[0].shape[0])] = 0
upper_mask = np.bool_(upper_mask)
# Returns the mask of the not available ROIs.
self.nan_mask = nan_mask
return self.nan_mask
def test_index_bool( self ):
a = np.arange(5)
x = np.bool_(False)
y = np.bool_(True)
z = a[x:y]
assert_equal( z, [0] )
def test_dmat_add(self):
assert_equals(self.pos_vec + self.neg_vec, self.unknown_vec)
result = Sign(np.bool_(True), self.arr)
assert_equals(self.pos_vec + Sign.NEGATIVE, result)
assert_equals(self.neg_vec + self.zero_vec, self.neg_vec)
result = Sign(np.bool_(True), ~self.true_mat)
assert_equals(self.neg_mat + Sign.NEGATIVE, result)
assert_equals(self.pos_vec + self.pos_vec, self.pos_vec)
assert_equals(self.unknown_mat + self.zero_mat, self.unknown_mat)
assert_equals(Sign.UNKNOWN + self.pos_mat, Sign.UNKNOWN)
def test_astype(self):
import numpy as np
a = np.bool_(True).astype(np.float32)
assert type(a) is np.float32
assert a == 1.0
a = np.bool_(True).astype('int32')
assert type(a) is np.int32
assert a == 1
a = np.str_('123').astype('int32')
assert type(a) is np.int32
assert a == 123
def test_mask_ratio(self):
self.assertEqual(mask_ratio(True), 1)
self.assertEqual(mask_ratio(np.bool_(True)), 1)
self.assertEqual(mask_ratio(False), 0)
self.assertEqual(mask_ratio(np.bool_(False)), 0)
array = np.ma.array(range(10))
self.assertEqual(mask_ratio(np.ma.getmaskarray(array)), 0)
array[0] = np.ma.masked
self.assertEqual(mask_ratio(np.ma.getmaskarray(array)), 0.1)
invalid_array = np.ma.array(range(100), mask=[True]*100)
self.assertEqual(mask_ratio(np.ma.getmaskarray(invalid_array)), 1)
def __init__(self, log_parameters=[], **kwargs):
# Runs Individual.__init__ for default object initialization.
super(Plankton, self).__init__(**kwargs)
# Some parameters and initializations
self.state_parameters = ['t', 'x', 'y', 'z', 'P', 'N_P']
# ... history list
self._attributes['history'] = dict()
# ... log list
n = self.size()
extra = {key: [nan] * n for key in log_parameters}
# Start history log
self.append_history(extra=extra)
# Assumes every individual is inside the model domain
self.in_domain = bool_(self.x) | bool_(self.y)
def construct_4d_adjacency_list(mask, numx=1, numy=1, numz=1, numt=1, nt=0):
"""
Basically the prepare_adj function from regreg, but with less options.
"""
regions = np.zeros(mask.shape)
regions.shape = mask.shape
reg_values = np.unique(regions)
vmap = np.cumsum(mask).reshape(mask.shape)
mask = np.bool_(mask.copy())
vmap[~mask] = -1
vmap -= 1 # sets vmap's values from 0 to mask.sum()-1
adj = []
nx, ny, nz = mask.shape
for i, j, k, t in itertools.product(range(nx), range(ny),
range(nz), range(nt)):
if mask[i, j, k, t]:
local_map = vmap[max((i-numx), 0):(i+numx+1),
max((j-numy), 0):(j+numy+1),
max((k-numz), 0):(k+numz+1),
max((t-numt), 0):(t+numt+1)]
local_reg = regions[max((i-numx), 0):(i+numx+1),
max((j-numy), 0):(j+numy+1),
max((k-numz), 0):(k+numz+1),
max((t-numt), 0):(t+numt+1)]
region = regions[i, j, k, t]
ind = (local_map > -1) * (local_reg == region)
ind = np.bool_(ind)
nbrs = np.array(local_map[ind], dtype=np.int)
adj.append(nbrs)
for i, a in enumerate(adj):
a[np.equal(a, i)] = -1
num_ind = np.max([len(a) for a in adj])
adjarray = -np.ones((len(adj), num_ind), dtype=np.int)
for i in range(len(adj)):
for j in range(len(adj[i])):
adjarray[i,j] = adj[i][j]
return adjarray
def test_local_time_constraint_utc():
time = Time('2001-02-03 04:05:06')
subaru = Observer.at_site("Subaru")
constraint = LocalTimeConstraint(min=dt.time(23, 50), max=dt.time(4, 8))
is_constraint_met = constraint(subaru, None, times=time)
assert is_constraint_met is np.bool_(True)
constraint = LocalTimeConstraint(min=dt.time(0, 2), max=dt.time(4, 3))
is_constraint_met = constraint(subaru, None, times=time)
assert is_constraint_met is np.bool_(False)
constraint = LocalTimeConstraint(min=dt.time(3, 8), max=dt.time(5, 35))
is_constraint_met = constraint(subaru, None, times=time)
assert is_constraint_met is np.bool_(True)
def test_numpy_scalar_conversion_values(self):
self.assertEqual(nd.as_py(nd.array(np.bool_(True))), True)
self.assertEqual(nd.as_py(nd.array(np.bool_(False))), False)
self.assertEqual(nd.as_py(nd.array(np.int8(100))), 100)
self.assertEqual(nd.as_py(nd.array(np.int8(-100))), -100)
self.assertEqual(nd.as_py(nd.array(np.int16(20000))), 20000)
self.assertEqual(nd.as_py(nd.array(np.int16(-20000))), -20000)
self.assertEqual(nd.as_py(nd.array(np.int32(1000000000))), 1000000000)
self.assertEqual(nd.as_py(nd.array(np.int64(-1000000000000))),
-1000000000000)
self.assertEqual(nd.as_py(nd.array(np.int64(1000000000000))),
1000000000000)
self.assertEqual(nd.as_py(nd.array(np.int32(-1000000000))),
-1000000000)
self.assertEqual(nd.as_py(nd.array(np.uint8(200))), 200)
self.assertEqual(nd.as_py(nd.array(np.uint16(50000))), 50000)
self.assertEqual(nd.as_py(nd.array(np.uint32(3000000000))), 3000000000)
self.assertEqual(nd.as_py(nd.array(np.uint64(10000000000000000000))),
10000000000000000000)
self.assertEqual(nd.as_py(nd.array(np.float32(2.5))), 2.5)
self.assertEqual(nd.as_py(nd.array(np.float64(2.5))), 2.5)
self.assertEqual(nd.as_py(nd.array(np.complex64(2.5-1j))), 2.5-1j)
self.assertEqual(nd.as_py(nd.array(np.complex128(2.5-1j))), 2.5-1j)
if np.__version__ >= '1.7':
# Various date units
self.assertEqual(nd.as_py(nd.array(np.datetime64('2000'))),
date(2000, 1, 1))
self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12'))),
date(2000, 12, 1))
self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13'))),
date(2000, 12, 13))
# Various datetime units
self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13T12Z'))),
datetime(2000, 12, 13, 12, 0))
self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13T12:30Z'))),
datetime(2000, 12, 13, 12, 30))
self.assertEqual(nd.as_py(nd.array(np.datetime64('1823-12-13T12:30Z'))),
datetime(1823, 12, 13, 12, 30))
self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13T12:30:24Z'))),
datetime(2000, 12, 13, 12, 30, 24))
self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13T12:30:24.123Z'))),
datetime(2000, 12, 13, 12, 30, 24, 123000))
self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13T12:30:24.123456Z'))),
datetime(2000, 12, 13, 12, 30, 24, 123456))
self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13T12:30:24.123456124Z'))),
datetime(2000, 12, 13, 12, 30, 24, 123456))
self.assertEqual(str(nd.array(np.datetime64('2000-12-13T12:30:24.123456124Z'))),
'2000-12-13T12:30:24.1234561Z')
self.assertEqual(str(nd.array(np.datetime64('1842-12-13T12:30:24.123456124Z'))),
'1842-12-13T12:30:24.1234561Z')
def __init__(self, endog, exog, censors):
self.nobs = float(np.shape(exog)[0])
self.endog = endog.reshape(self.nobs, 1)
self.exog = exog.reshape(self.nobs, -1)
self.censors = censors.reshape(self.nobs, 1)
self.nvar = self.exog.shape[1]
idx = np.lexsort((-self.censors[:, 0], self.endog[:, 0]))
self.endog = self.endog[idx]
self.exog = self.exog[idx]
self.censors = self.censors[idx]
self.censors[-1] = 1 # Sort in init, not in function
self.uncens_nobs = np.sum(self.censors)
self.uncens_endog = self.endog[np.bool_(self.censors), :].\
reshape(-1, 1)
self.uncens_exog = self.exog[np.bool_(self.censors.flatten()), :]
def sign(constant):
if isinstance(constant, numbers.Number):
return Sign(np.bool_(constant < 0), np.bool_(constant > 0))
elif isinstance(constant, cvxopt.spmatrix):
# Convert to COO matrix.
V = np.array(list(constant.V))
I = list(constant.I)
J = list(constant.J)
# Check if entries > 0 for pos_mat, < 0 for neg_mat.
neg_mat = sp.coo_matrix((V < 0,(I,J)), shape=constant.size, dtype='bool')
pos_mat = sp.coo_matrix((V > 0,(I,J)), shape=constant.size, dtype='bool')
return Sign(SparseBoolMat(neg_mat), SparseBoolMat(pos_mat))
else:
mat = INTERFACES[np.ndarray].const_to_matrix(constant)
return Sign(mat < 0, mat > 0)
def test_operator_deprecation(self):
array = np.array([True])
generic = np.bool_(True)
# Minus operator/subtract ufunc:
self.assert_deprecated(operator.sub, args=(array, array))
self.assert_deprecated(operator.sub, args=(generic, generic))
def test_array_flat_setitem(self):
# Test indexing of array.flat object
pyfunc = array_flat_setitem
def check(arr, ind):
arrty = typeof(arr)
cr = self.ccache.compile(pyfunc, (arrty, typeof(ind), arrty.dtype))
# Use np.copy() to keep the layout
expected = np.copy(arr)
got = np.copy(arr)
pyfunc(expected, ind, 123)
cr.entry_point(got, ind, 123)
self.assertPreciseEqual(got, expected)
arr = np.arange(24).reshape(4, 2, 3)
for i in range(arr.size):
check(arr, i)
arr = arr.T
for i in range(arr.size):
check(arr, i)
arr = arr[::2]
for i in range(arr.size):
check(arr, i)
arr = np.array([42]).reshape(())
for i in range(arr.size):
check(arr, i)
# Boolean array
arr = np.bool_([1, 0, 0, 1])
for i in range(arr.size):
check(arr, i)
arr = arr[::2]
for i in range(arr.size):
check(arr, i)
def test_bools(self):
arr = np.array([True, False, True, True, True], dtype='O')
result = lib.infer_dtype(arr)
self.assertEqual(result, 'boolean')
arr = np.array([np.bool_(True), np.bool_(False)], dtype='O')
result = lib.infer_dtype(arr)
self.assertEqual(result, 'boolean')
arr = np.array([True, False, True, 'foo'], dtype='O')
result = lib.infer_dtype(arr)
self.assertEqual(result, 'mixed')
arr = np.array([True, False, True], dtype=bool)
result = lib.infer_dtype(arr)
self.assertEqual(result, 'boolean')
def test_array_ndenumerate_2d(self):
arr = np.arange(12).reshape(4, 3)
arrty = typeof(arr)
self.assertEqual(arrty.ndim, 2)
self.assertEqual(arrty.layout, 'C')
self.assertTrue(arr.flags.c_contiguous)
# Test with C-contiguous array
self.check_array_ndenumerate_sum(arr, arrty)
# Test with Fortran-contiguous array
arr = arr.transpose()
self.assertFalse(arr.flags.c_contiguous)
self.assertTrue(arr.flags.f_contiguous)
arrty = typeof(arr)
self.assertEqual(arrty.layout, 'F')
self.check_array_ndenumerate_sum(arr, arrty)
# Test with non-contiguous array
arr = arr[::2]
self.assertFalse(arr.flags.c_contiguous)
self.assertFalse(arr.flags.f_contiguous)
arrty = typeof(arr)
self.assertEqual(arrty.layout, 'A')
self.check_array_ndenumerate_sum(arr, arrty)
# Boolean array
arr = np.bool_([1, 0, 0, 1]).reshape((2, 2))
self.check_array_ndenumerate_sum(arr, typeof(arr))
def check_nonzero(self, pyfunc):
def fac(N):
np.random.seed(42)
arr = np.random.random(N)
arr[arr < 0.3] = 0.0
arr[arr > 0.7] = float('nan')
return arr
def check_arr(arr):
cres = compile_isolated(pyfunc, (typeof(arr),))
expected = pyfunc(arr)
# NOTE: Numpy 1.9 returns readonly arrays for multidimensional
# arrays. Workaround this by copying the results.
expected = [a.copy() for a in expected]
self.assertPreciseEqual(cres.entry_point(arr), expected)
arr = np.int16([1, 0, -1, 0])
check_arr(arr)
arr = np.bool_([1, 0, 1])
check_arr(arr)
arr = fac(24)
check_arr(arr)
check_arr(arr.reshape((3, 8)))
check_arr(arr.reshape((3, 8)).T)
check_arr(arr.reshape((3, 8))[::2])
check_arr(arr.reshape((2, 3, 4)))
check_arr(arr.reshape((2, 3, 4)).T)
check_arr(arr.reshape((2, 3, 4))[::2])
for v in (0.0, 1.5, float('nan')):
arr = np.array([v]).reshape(())
check_arr(arr)
def test_array_flat_getitem(self):
# Test indexing of array.flat object
pyfunc = array_flat_getitem
def check(arr, ind):
cr = self.ccache.compile(pyfunc, (typeof(arr), typeof(ind)))
expected = pyfunc(arr, ind)
self.assertEqual(cr.entry_point(arr, ind), expected)
arr = np.arange(24).reshape(4, 2, 3)
for i in range(arr.size):
check(arr, i)
arr = arr.T
for i in range(arr.size):
check(arr, i)
arr = arr[::2]
for i in range(arr.size):
check(arr, i)
arr = np.array([42]).reshape(())
for i in range(arr.size):
check(arr, i)
# Boolean array
arr = np.bool_([1, 0, 0, 1])
for i in range(arr.size):
check(arr, i)
arr = arr[::2]
for i in range(arr.size):
check(arr, i)
def test_is_number(self):
self.assertTrue(is_number(True))
self.assertTrue(is_number(1))
self.assertTrue(is_number(1.1))
self.assertTrue(is_number(1 + 3j))
self.assertTrue(is_number(np.bool(False)))
self.assertTrue(is_number(np.int64(1)))
self.assertTrue(is_number(np.float64(1.1)))
self.assertTrue(is_number(np.complex128(1 + 3j)))
self.assertTrue(is_number(np.nan))
self.assertFalse(is_number(None))
self.assertFalse(is_number('x'))
self.assertFalse(is_number(datetime(2011, 1, 1)))
self.assertFalse(is_number(np.datetime64('2011-01-01')))
self.assertFalse(is_number(Timestamp('2011-01-01')))
self.assertFalse(is_number(Timestamp('2011-01-01',
tz='US/Eastern')))
self.assertFalse(is_number(timedelta(1000)))
self.assertFalse(is_number(Timedelta('1 days')))
# questionable
self.assertFalse(is_number(np.bool_(False)))
self.assertTrue(is_number(np.timedelta64(1, 'D')))
def test_unary_minus_operator_deprecation(self):
array = np.array([True])
generic = np.bool_(True)
# Unary minus/negative ufunc:
self.assert_deprecated(operator.neg, args=(array,))
self.assert_deprecated(operator.neg, args=(generic,))
def test_base_json_conv(self):
assert isinstance(base_json_conv(numpy.bool_(1)), bool) is True
assert isinstance(base_json_conv(numpy.int64(1)), int) is True
assert isinstance(base_json_conv(set([1])), list) is True
assert isinstance(base_json_conv(Decimal('1.0')), float) is True
assert isinstance(base_json_conv(uuid.uuid4()), str) is True
def test_np_where_3(self):
pyfunc = np_where_3
def fac(N):
np.random.seed(42)
arr = np.random.random(N)
arr[arr < 0.3] = 0.0
arr[arr > 0.7] = float('nan')
return arr
layouts = cycle(['C', 'F', 'A'])
_types = [np.int32, np.int64, np.float32, np.float64, np.complex64,
np.complex128]
np.random.seed(42)
def check_arr(arr, layout=False):
np.random.shuffle(_types)
if layout != False:
x = np.zeros_like(arr, dtype=_types[0], order=layout)
y = np.zeros_like(arr, dtype=_types[1], order=layout)
arr = arr.copy(order=layout)
else:
x = np.zeros_like(arr, dtype=_types[0], order=next(layouts))
y = np.zeros_like(arr, dtype=_types[1], order=next(layouts))
x.fill(4)
y.fill(9)
cres = compile_isolated(pyfunc, (typeof(arr), typeof(x), typeof(y)))
expected = pyfunc(arr, x, y)
got = cres.entry_point(arr, x, y)
# Contiguity of result varies across Numpy versions, only
# check contents. NumPy 1.11+ seems to stabilize.
if numpy_version < (1, 11):
self.assertEqual(got.dtype, expected.dtype)
np.testing.assert_array_equal(got, expected)
else:
self.assertPreciseEqual(got, expected)
def check_scal(scal):
x = 4
y = 5
np.random.shuffle(_types)
x = _types[0](4)
y = _types[1](5)
cres = compile_isolated(pyfunc, (typeof(scal), typeof(x), typeof(y)))
expected = pyfunc(scal, x, y)
got = cres.entry_point(scal, x, y)
self.assertPreciseEqual(got, expected)
arr = np.int16([1, 0, -1, 0])
check_arr(arr)
arr = np.bool_([1, 0, 1])
check_arr(arr)
arr = fac(24)
check_arr(arr)
check_arr(arr.reshape((3, 8)))
check_arr(arr.reshape((3, 8)).T)
check_arr(arr.reshape((3, 8))[::2])
check_arr(arr.reshape((2, 3, 4)))
check_arr(arr.reshape((2, 3, 4)).T)
check_arr(arr.reshape((2, 3, 4))[::2])
check_arr(arr.reshape((2, 3, 4)), layout='F')
check_arr(arr.reshape((2, 3, 4)).T, layout='F')
check_arr(arr.reshape((2, 3, 4))[::2], layout='F')
for v in (0.0, 1.5, float('nan')):
arr = np.array([v]).reshape(())
check_arr(arr)
for x in (0, 1, True, False, 2.5, 0j):
check_scal(x)
def test_table_typing_numpy():
# Pulled from https://numpy.org/devdocs/user/basics.types.html
# Numerics
table = wandb.Table(columns=["A"], dtype=[NumberType])
table.add_data(None)
table.add_data(42)
table.add_data(np.byte(1))
table.add_data(np.short(42))
table.add_data(np.ushort(42))
table.add_data(np.intc(42))
table.add_data(np.uintc(42))
table.add_data(np.int_(42))
table.add_data(np.uint(42))
table.add_data(np.longlong(42))
table.add_data(np.ulonglong(42))
table.add_data(np.half(42))
table.add_data(np.float16(42))
table.add_data(np.single(42))
table.add_data(np.double(42))
table.add_data(np.longdouble(42))
table.add_data(np.csingle(42))
table.add_data(np.cdouble(42))
table.add_data(np.clongdouble(42))
table.add_data(np.int8(42))
table.add_data(np.int16(42))
table.add_data(np.int32(42))
table.add_data(np.int64(42))
table.add_data(np.uint8(42))
table.add_data(np.uint16(42))
table.add_data(np.uint32(42))
table.add_data(np.uint64(42))
table.add_data(np.intp(42))
table.add_data(np.uintp(42))
table.add_data(np.float32(42))
table.add_data(np.float64(42))
table.add_data(np.float_(42))
table.add_data(np.complex64(42))
table.add_data(np.complex128(42))
table.add_data(np.complex_(42))
# Booleans
table = wandb.Table(columns=["A"], dtype=[BooleanType])
table.add_data(None)
table.add_data(True)
table.add_data(False)
table.add_data(np.bool_(True))
# Array of Numerics
table = wandb.Table(columns=["A"], dtype=[[NumberType]])
table.add_data(None)
table.add_data([42])
table.add_data(np.array([1, 0], dtype=np.byte))
table.add_data(np.array([42, 42], dtype=np.short))
table.add_data(np.array([42, 42], dtype=np.ushort))
table.add_data(np.array([42, 42], dtype=np.intc))
table.add_data(np.array([42, 42], dtype=np.uintc))
table.add_data(np.array([42, 42], dtype=np.int_))
table.add_data(np.array([42, 42], dtype=np.uint))
table.add_data(np.array([42, 42], dtype=np.longlong))
table.add_data(np.array([42, 42], dtype=np.ulonglong))
table.add_data(np.array([42, 42], dtype=np.half))
table.add_data(np.array([42, 42], dtype=np.float16))
table.add_data(np.array([42, 42], dtype=np.single))
table.add_data(np.array([42, 42], dtype=np.double))
table.add_data(np.array([42, 42], dtype=np.longdouble))
table.add_data(np.array([42, 42], dtype=np.csingle))
table.add_data(np.array([42, 42], dtype=np.cdouble))
table.add_data(np.array([42, 42], dtype=np.clongdouble))
table.add_data(np.array([42, 42], dtype=np.int8))
table.add_data(np.array([42, 42], dtype=np.int16))
table.add_data(np.array([42, 42], dtype=np.int32))
table.add_data(np.array([42, 42], dtype=np.int64))
table.add_data(np.array([42, 42], dtype=np.uint8))
table.add_data(np.array([42, 42], dtype=np.uint16))
table.add_data(np.array([42, 42], dtype=np.uint32))
table.add_data(np.array([42, 42], dtype=np.uint64))
table.add_data(np.array([42, 42], dtype=np.intp))
table.add_data(np.array([42, 42], dtype=np.uintp))
table.add_data(np.array([42, 42], dtype=np.float32))
table.add_data(np.array([42, 42], dtype=np.float64))
table.add_data(np.array([42, 42], dtype=np.float_))
table.add_data(np.array([42, 42], dtype=np.complex64))
table.add_data(np.array([42, 42], dtype=np.complex128))
table.add_data(np.array([42, 42], dtype=np.complex_))
# Array of Booleans
table = wandb.Table(columns=["A"], dtype=[[BooleanType]])
table.add_data(None)
table.add_data([True])
table.add_data([False])
table.add_data(np.array([True, False], dtype=np.bool_))
# Nested arrays
table = wandb.Table(columns=["A"])
table.add_data([[[[1, 2, 3]]]])
table.add_data(np.array([[[[1, 2, 3]]]]))
def test_recursively_convert_to_json_serializable(self):
D = ge.dataset.PandasDataset({
'x': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
})
D.expect_column_values_to_be_in_set("x",
set([1, 2, 3, 4, 5, 6, 7, 8, 9]),
mostly=.8)
part = ge.dataset.util.partition_data(D.x)
D.expect_column_kl_divergence_to_be_less_than("x", part, .6)
# Dumping this JSON object verifies that everything is serializable
json.dumps(D.get_expectations_config(), indent=2)
x = {
'w': ["aaaa", "bbbb", 1.3, 5, 6, 7],
'x': np.array([1, 2, 3]),
'y': {
'alpha': None,
'beta': np.nan,
'delta': np.inf,
'gamma': -np.inf
},
'z': set([1, 2, 3, 4, 5]),
'zz': (1, 2, 3),
'zzz': [
datetime.datetime(2017, 1, 1),
datetime.date(2017, 5, 1),
],
'np.bool': np.bool_([True, False, True]),
'np.int_': np.int_([5, 3, 2]),
'np.int8': np.int8([5, 3, 2]),
'np.int16': np.int16([10, 6, 4]),
'np.int32': np.int32([20, 12, 8]),
'np.uint': np.uint([20, 5, 6]),
'np.uint8': np.uint8([40, 10, 12]),
'np.uint64': np.uint64([80, 20, 24]),
'np.float_': np.float_([3.2, 5.6, 7.8]),
'np.float32': np.float32([5.999999999, 5.6]),
'np.float64': np.float64([5.9999999999999999999, 10.2]),
'np.float128': np.float128([5.999999999998786324399999999, 20.4]),
# 'np.complex64': np.complex64([10.9999999 + 4.9999999j, 11.2+7.3j]),
# 'np.complex128': np.complex128([20.999999999978335216827+10.99999999j, 22.4+14.6j]),
# 'np.complex256': np.complex256([40.99999999 + 20.99999999j, 44.8+29.2j]),
'np.str': np.unicode_(["hello"]),
'yyy': decimal.Decimal(123.456)
}
x = ge.dataset.util.recursively_convert_to_json_serializable(x)
self.assertEqual(type(x['x']), list)
self.assertEqual(type(x['np.bool'][0]), bool)
self.assertEqual(type(x['np.int_'][0]), int)
self.assertEqual(type(x['np.int8'][0]), int)
self.assertEqual(type(x['np.int16'][0]), int)
self.assertEqual(type(x['np.int32'][0]), int)
# Integers in python 2.x can be of type int or of type long
if sys.version_info.major >= 3:
# Python 3.x
self.assertTrue(isinstance(x['np.uint'][0], int))
self.assertTrue(isinstance(x['np.uint8'][0], int))
self.assertTrue(isinstance(x['np.uint64'][0], int))
elif sys.version_info.major >= 2:
# Python 2.x
self.assertTrue(isinstance(x['np.uint'][0], (int, long)))
self.assertTrue(isinstance(x['np.uint8'][0], (int, long)))
self.assertTrue(isinstance(x['np.uint64'][0], (int, long)))
self.assertEqual(type(x['np.float32'][0]), float)
self.assertEqual(type(x['np.float64'][0]), float)
self.assertEqual(type(x['np.float128'][0]), float)
# self.assertEqual(type(x['np.complex64'][0]), complex)
# self.assertEqual(type(x['np.complex128'][0]), complex)
# self.assertEqual(type(x['np.complex256'][0]), complex)
self.assertEqual(type(x['np.float_'][0]), float)
# Make sure nothing is going wrong with precision rounding
# self.assertAlmostEqual(x['np.complex128'][0].real, 20.999999999978335216827, places=sys.float_info.dig)
self.assertAlmostEqual(x['np.float128'][0],
5.999999999998786324399999999,
places=sys.float_info.dig)
# TypeError when non-serializable numpy object is in dataset.
with self.assertRaises(TypeError):
y = {'p': np.DataSource()}
ge.dataset.util.recursively_convert_to_json_serializable(y)
try:
x = unicode("abcdefg")
x = ge.dataset.util.recursively_convert_to_json_serializable(x)
self.assertEqual(type(x), unicode)
except NameError:
pass
# Default imports
import scipy.stats as stats
import pandas as pd
import numpy as np
from statsmodels.stats.weightstats import ztest
df = pd.read_csv('data/house_pricing.csv')
bol = np.bool_(dtype=bool)
# Enter Code Here
def t_statistic(data):
#z_statistic, p_value = ztest(x1=data[data['Neighborhood'] == 'OldTown']['GrLivArea'], value=data['GrLivArea'].mean())
z_statistic, p_value = stats.ttest_1samp(
a=data[data['Neighborhood'] == 'OldTown']['GrLivArea'],
popmean=data['GrLivArea'].mean())
if z_statistic <= 0.05:
bol = np.bool_(False)
else:
bol = np.bool_(True)
return p_value, bol
def __init__(self):
self.been_called = False
self.ncalls = 0
self.testres = [False, 'force accept', True, np.bool_(True),
np.bool_(False), [], {}, 0, 1]
from typing import Any
import numpy as np
f8 = np.float64()
i8 = np.int64()
u8 = np.uint64()
f4 = np.float32()
i4 = np.int32()
u4 = np.uint32()
td = np.timedelta64(0, "D")
b_ = np.bool_()
b = bool()
f = float()
i = int()
AR_b: np.ndarray[Any, np.dtype[np.bool_]]
AR_m: np.ndarray[Any, np.dtype[np.timedelta64]]
# Time structures
reveal_type(td % td) # E: numpy.timedelta64
reveal_type(AR_m % td) # E: Any
reveal_type(td % AR_m) # E: Any
reveal_type(divmod(td, td)) # E: Tuple[{int64}, numpy.timedelta64]
reveal_type(
divmod(AR_m, td)
) # E: Tuple[numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]], numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]]]
def test_numpy_scalars_ok(self, all_logical_operators):
a = pd.array([True, False, None], dtype="boolean")
op = getattr(a, all_logical_operators)
tm.assert_extension_array_equal(op(True), op(np.bool_(True)))
tm.assert_extension_array_equal(op(False), op(np.bool_(False)))
if arraysize != []:
converter = converter.array_type(
field, converter, arraysize, config)
if not fixed:
converter = converter.vararray_type(
field, converter, arraysize, config)
return converter
numpy_dtype_to_field_mapping = {
np.float64().dtype.num : 'double',
np.float32().dtype.num : 'float',
np.bool_().dtype.num : 'bit',
np.uint8().dtype.num : 'unsignedByte',
np.int16().dtype.num : 'short',
np.int32().dtype.num : 'int',
np.int64().dtype.num : 'long',
np.complex64().dtype.num : 'floatComplex',
np.complex128().dtype.num : 'doubleComplex',
np.unicode_().dtype.num : 'unicodeChar'
}
numpy_dtype_to_field_mapping[np.bytes_().dtype.num] = 'char'
def _all_bytes(column):
for x in column:
class TestLogicalOps(BaseOpsUtil):
def test_numpy_scalars_ok(self, all_logical_operators):
a = pd.array([True, False, None], dtype="boolean")
op = getattr(a, all_logical_operators)
tm.assert_extension_array_equal(op(True), op(np.bool_(True)))
tm.assert_extension_array_equal(op(False), op(np.bool_(False)))
def get_op_from_name(self, op_name):
short_opname = op_name.strip("_")
short_opname = short_opname if "xor" in short_opname else short_opname + "_"
try:
op = getattr(operator, short_opname)
except AttributeError:
# Assume it is the reverse operator
rop = getattr(operator, short_opname[1:])
op = lambda x, y: rop(y, x)
return op
def test_empty_ok(self, all_logical_operators):
a = pd.array([], dtype="boolean")
op_name = all_logical_operators
result = getattr(a, op_name)(True)
tm.assert_extension_array_equal(a, result)
result = getattr(a, op_name)(False)
tm.assert_extension_array_equal(a, result)
result = getattr(a, op_name)(pd.NA)
tm.assert_extension_array_equal(a, result)
def test_logical_length_mismatch_raises(self, all_logical_operators):
op_name = all_logical_operators
a = pd.array([True, False, None], dtype="boolean")
msg = "Lengths must match to compare"
with pytest.raises(ValueError, match=msg):
getattr(a, op_name)([True, False])
with pytest.raises(ValueError, match=msg):
getattr(a, op_name)(np.array([True, False]))
with pytest.raises(ValueError, match=msg):
getattr(a, op_name)(pd.array([True, False], dtype="boolean"))
def test_logical_nan_raises(self, all_logical_operators):
op_name = all_logical_operators
a = pd.array([True, False, None], dtype="boolean")
msg = "Got float instead"
with pytest.raises(TypeError, match=msg):
getattr(a, op_name)(np.nan)
@pytest.mark.parametrize("other", ["a", 1])
def test_non_bool_or_na_other_raises(self, other, all_logical_operators):
a = pd.array([True, False], dtype="boolean")
with pytest.raises(TypeError, match=str(type(other).__name__)):
getattr(a, all_logical_operators)(other)
def test_kleene_or(self):
# A clear test of behavior.
a = pd.array([True] * 3 + [False] * 3 + [None] * 3, dtype="boolean")
b = pd.array([True, False, None] * 3, dtype="boolean")
result = a | b
expected = pd.array(
[True, True, True, True, False, None, True, None, None], dtype="boolean"
)
tm.assert_extension_array_equal(result, expected)
result = b | a
tm.assert_extension_array_equal(result, expected)
# ensure we haven't mutated anything inplace
tm.assert_extension_array_equal(
a, pd.array([True] * 3 + [False] * 3 + [None] * 3, dtype="boolean")
)
tm.assert_extension_array_equal(
b, pd.array([True, False, None] * 3, dtype="boolean")
)
@pytest.mark.parametrize(
"other, expected",
[
(pd.NA, [True, None, None]),
(True, [True, True, True]),
(np.bool_(True), [True, True, True]),
(False, [True, False, None]),
(np.bool_(False), [True, False, None]),
],
)
def test_kleene_or_scalar(self, other, expected):
# TODO: test True & False
a = pd.array([True, False, None], dtype="boolean")
result = a | other
expected = pd.array(expected, dtype="boolean")
tm.assert_extension_array_equal(result, expected)
result = other | a
tm.assert_extension_array_equal(result, expected)
# ensure we haven't mutated anything inplace
tm.assert_extension_array_equal(
a, pd.array([True, False, None], dtype="boolean")
)
def test_kleene_and(self):
# A clear test of behavior.
a = pd.array([True] * 3 + [False] * 3 + [None] * 3, dtype="boolean")
b = pd.array([True, False, None] * 3, dtype="boolean")
result = a & b
expected = pd.array(
[True, False, None, False, False, False, None, False, None], dtype="boolean"
)
tm.assert_extension_array_equal(result, expected)
result = b & a
tm.assert_extension_array_equal(result, expected)
# ensure we haven't mutated anything inplace
tm.assert_extension_array_equal(
a, pd.array([True] * 3 + [False] * 3 + [None] * 3, dtype="boolean")
)
tm.assert_extension_array_equal(
b, pd.array([True, False, None] * 3, dtype="boolean")
)
@pytest.mark.parametrize(
"other, expected",
[
(pd.NA, [None, False, None]),
(True, [True, False, None]),
(False, [False, False, False]),
(np.bool_(True), [True, False, None]),
(np.bool_(False), [False, False, False]),
],
)
def test_kleene_and_scalar(self, other, expected):
a = pd.array([True, False, None], dtype="boolean")
result = a & other
expected = pd.array(expected, dtype="boolean")
tm.assert_extension_array_equal(result, expected)
result = other & a
tm.assert_extension_array_equal(result, expected)
# ensure we haven't mutated anything inplace
tm.assert_extension_array_equal(
a, pd.array([True, False, None], dtype="boolean")
)
def test_kleene_xor(self):
a = pd.array([True] * 3 + [False] * 3 + [None] * 3, dtype="boolean")
b = pd.array([True, False, None] * 3, dtype="boolean")
result = a ^ b
expected = pd.array(
[False, True, None, True, False, None, None, None, None], dtype="boolean"
)
tm.assert_extension_array_equal(result, expected)
result = b ^ a
tm.assert_extension_array_equal(result, expected)
# ensure we haven't mutated anything inplace
tm.assert_extension_array_equal(
a, pd.array([True] * 3 + [False] * 3 + [None] * 3, dtype="boolean")
)
tm.assert_extension_array_equal(
b, pd.array([True, False, None] * 3, dtype="boolean")
)
@pytest.mark.parametrize(
"other, expected",
[
(pd.NA, [None, None, None]),
(True, [False, True, None]),
(np.bool_(True), [False, True, None]),
(np.bool_(False), [True, False, None]),
],
)
def test_kleene_xor_scalar(self, other, expected):
a = pd.array([True, False, None], dtype="boolean")
result = a ^ other
expected = pd.array(expected, dtype="boolean")
tm.assert_extension_array_equal(result, expected)
result = other ^ a
tm.assert_extension_array_equal(result, expected)
# ensure we haven't mutated anything inplace
tm.assert_extension_array_equal(
a, pd.array([True, False, None], dtype="boolean")
)
@pytest.mark.parametrize("other", [True, False, pd.NA, [True, False, None] * 3])
def test_no_masked_assumptions(self, other, all_logical_operators):
# The logical operations should not assume that masked values are False!
a = pd.arrays.BooleanArray(
np.array([True, True, True, False, False, False, True, False, True]),
np.array([False] * 6 + [True, True, True]),
)
b = pd.array([True] * 3 + [False] * 3 + [None] * 3, dtype="boolean")
if isinstance(other, list):
other = pd.array(other, dtype="boolean")
result = getattr(a, all_logical_operators)(other)
expected = getattr(b, all_logical_operators)(other)
tm.assert_extension_array_equal(result, expected)
if isinstance(other, BooleanArray):
other._data[other._mask] = True
a._data[a._mask] = False
result = getattr(a, all_logical_operators)(other)
expected = getattr(b, all_logical_operators)(other)
tm.assert_extension_array_equal(result, expected)
def test_environment_stepper_component_with_large_impala_architecture(
self):
env_spec = dict(type="deepmind_lab",
level_id="seekavoid_arena_01",
observations=["RGB_INTERLEAVED", "INSTR"],
frameskip=4)
dummy_env = Environment.from_spec(env_spec)
state_space = dummy_env.state_space
action_space = dummy_env.action_space
actor_component = ActorComponent(
# Preprocessor spec (only for image and prev-action channel).
dict(
type="dict-preprocessor-stack",
preprocessors=dict(
## The images from the env are divided by 255.
#RGB_INTERLEAVED=[dict(type="divide", divisor=255)],
# The prev. action/reward from the env must be flattened/bumped-up-to-(1,).
previous_action=[
dict(type="reshape",
flatten=True,
flatten_categories=action_space.num_categories)
],
previous_reward=[
dict(type="reshape", new_shape=(1, )),
dict(type="convert_type", to_dtype="float32")
],
)),
# Policy spec.
dict(network_spec=LargeIMPALANetwork(), action_space=action_space),
# Exploration spec.
Exploration(epsilon_spec=dict(decay_spec=dict(type="linear_decay",
from_=1.0,
to_=0.1,
start_timestep=0,
num_timesteps=100))))
environment_stepper = EnvironmentStepper(
environment_spec=env_spec,
actor_component_spec=actor_component,
state_space=state_space,
reward_space="float32",
internal_states_space=self.internal_states_space,
num_steps=100,
# Add both prev-action and -reward into the state sent through the network.
add_previous_action_to_state=True,
add_previous_reward_to_state=True,
add_action_probs=True,
action_probs_space=self.action_probs_space)
test = ComponentTest(
component=environment_stepper,
action_space=action_space,
)
# Reset the stepper.
test.test("reset")
# Step n times through the Env and collect results.
# 1st return value is the step-op (None), 2nd return value is the tuple of items (3 steps each), with each
# step containing: Preprocessed state, actions, rewards, episode returns, terminals, (raw) next-states.
time_start = time.perf_counter()
steps = 10
for _ in range(steps):
out = test.test("step")
time_total = time.perf_counter() - time_start
print(
"Done running {}x{} steps in Deepmind Lab env using IMPALA network in {}sec ({} actions/sec)."
.format(steps, environment_stepper.num_steps, time_total,
environment_stepper.num_steps * steps / time_total))
# Check types of outputs.
self.assertTrue(isinstance(
out, DataOpTuple)) # the step results as a tuple (see below)
# Check types of single data.
self.assertTrue(out[0]["INSTR"].dtype == np.object)
self.assertTrue(out[0]["RGB_INTERLEAVED"].dtype == np.float32)
self.assertTrue(out[0]["RGB_INTERLEAVED"].min() >=
0.0) # make sure we have pixels / 255
self.assertTrue(out[0]["RGB_INTERLEAVED"].max() <= 1.0)
self.assertTrue(out[1].dtype == np.int32) # actions
self.assertTrue(out[2].dtype == np.float32) # rewards
self.assertTrue(out[3].dtype == np.float32) # episode return
self.assertTrue(out[4].dtype == np.bool_) # next-state is terminal?
self.assertTrue(out[5]["INSTR"].dtype ==
np.object) # next state (raw, not preprocessed)
self.assertTrue(out[5]["RGB_INTERLEAVED"].dtype ==
np.uint8) # next state (raw, not preprocessed)
self.assertTrue(
out[5]["RGB_INTERLEAVED"].min() >= 0) # make sure we have pixels
self.assertTrue(out[5]["RGB_INTERLEAVED"].max() <= 255)
# action probs (test whether sum to one).
self.assertTrue(out[6].dtype == np.float32)
self.assertTrue(out[6].min() >= 0.0)
self.assertTrue(out[6].max() <= 1.0)
recursive_assert_almost_equal(
out[6].sum(axis=-1, keepdims=False),
np.ones(shape=(environment_stepper.num_steps, )),
decimals=4)
# internal states (c- and h-state)
self.assertTrue(out[7][0].dtype == np.float32)
self.assertTrue(out[7][1].dtype == np.float32)
self.assertTrue(out[7][0].shape == (environment_stepper.num_steps,
256))
self.assertTrue(out[7][1].shape == (environment_stepper.num_steps,
256))
# Check whether episode returns match single rewards (including terminal signals).
episode_returns = 0.0
for i in range(environment_stepper.num_steps):
episode_returns += out[2][i]
self.assertAlmostEqual(episode_returns, out[3][i])
# Terminal: Reset for next step.
if out[4][i] is np.bool_(True):
episode_returns = 0.0
test.terminate()
def main(_):
# CHECK-LABEL: TEST: abs int32[]
# CHECK: mhlo.abs
# CHECK-SAME: tensor<i32>
print_ir(np.int32(0))(lax.abs)
# CHECK-LABEL: TEST: add float32[] float32[]
# CHECK: mhlo.add
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1), np.float32(2))(lax.add)
# CHECK-LABEL: TEST: acos float32[]
# CHECK: mhlo.atan2
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1))(lax.acos)
# CHECK-LABEL: TEST: acosh float32[]
# CHECK: chlo.acosh
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.acosh)
# CHECK-LABEL: TEST: asin float32[]
# CHECK: mhlo.atan2
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1))(lax.asin)
# CHECK-LABEL: TEST: asinh float32[]
# CHECK: chlo.asinh
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.asinh)
# CHECK-LABEL: TEST: atan float32[]
# CHECK: mhlo.atan2
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1))(lax.atan)
# CHECK-LABEL: TEST: atanh float32[]
# CHECK: chlo.atanh
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.atanh)
# CHECK-LABEL: TEST: atan2 float64[] float64[]
# CHECK: mhlo.atan2
# CHECK-SAME: tensor<f64>
print_ir(np.float64(1), np.float64(2))(lax.atan2)
# CHECK-LABEL: TEST: bessel_i0e float32[]
# CHECK: xla_fallback_bessel_i0e
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.bessel_i0e)
# CHECK-LABEL: TEST: bessel_i1e float32[]
# CHECK: xla_fallback_bessel_i1e
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.bessel_i1e)
# CHECK-LABEL: TEST: betainc float32[] float32[] float32[]
# CHECK: xla_fallback_regularized_incomplete_beta
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0), np.float32(0), np.float32(0))(lax.betainc)
# CHECK-LABEL: TEST: bitcast_convert_type uint32[7]
# CHECK: mhlo.bitcast_convert
# CHECK-SAME: tensor<7xui32>
# CHECK-SAME: tensor<7xf32>
print_ir(np.empty((7, ), np.uint32))(partial(lax.bitcast_convert_type,
new_dtype=np.float32))
# CHECK-LABEL: TEST: bitwise_and int32[] int32[]
# CHECK: mhlo.and
# CHECK-SAME: tensor<i32>
print_ir(np.int32(1), np.int32(2))(lax.bitwise_and)
# CHECK-LABEL: TEST: bitwise_and bool[] bool[]
# CHECK: mhlo.and
# CHECK-SAME: tensor<i1>
print_ir(np.bool_(0), np.bool_(0))(lax.bitwise_and)
# CHECK-LABEL: TEST: bitwise_or int32[] int32[]
# CHECK: mhlo.or
# CHECK-SAME: tensor<i32>
print_ir(np.int32(1), np.int32(2))(lax.bitwise_or)
# CHECK-LABEL: TEST: bitwise_or bool[] bool[]
# CHECK: mhlo.or
# CHECK-SAME: tensor<i1>
print_ir(np.bool_(0), np.bool_(0))(lax.bitwise_or)
# CHECK-LABEL: TEST: bitwise_xor int32[] int32[]
# CHECK: mhlo.xor
# CHECK-SAME: tensor<i32>
print_ir(np.int32(1), np.int32(2))(lax.bitwise_xor)
# CHECK-LABEL: TEST: bitwise_xor bool[] bool[]
# CHECK: mhlo.xor
# CHECK-SAME: tensor<i1>
print_ir(np.bool_(0), np.bool_(0))(lax.bitwise_xor)
# CHECK-LABEL: TEST: cbrt bfloat16[]
# CHECK: mhlo.cbrt
# CHECK-SAME: tensor<bf16>
print_ir(jnp.bfloat16(0))(lax.cbrt)
# CHECK-LABEL: TEST: clamp bfloat16[] bfloat16[] bfloat16[]
# CHECK: mhlo.clamp
# CHECK-SAME: tensor<bf16>
print_ir(jnp.bfloat16(0), jnp.bfloat16(0), jnp.bfloat16(0))(lax.clamp)
# CHECK-LABEL: TEST: ceil float16[7]
# CHECK: mhlo.ceil
# CHECK-SAME: tensor<7xf16>
print_ir(np.empty((7, ), np.float16))(lax.ceil)
# CHECK-LABEL: TEST: convert_element_type float16[7]
# CHECK: mhlo.convert
# CHECK-SAME: tensor<7xf16>
# CHECK-SAME: tensor<7xf32>
print_ir(np.empty((7, ), np.float16))(partial(lax.convert_element_type,
new_dtype=np.float32))
# CHECK-LABEL: TEST: convert_element_type complex64[7]
# CHECK: mhlo.real
# CHECK-SAME: tensor<7xcomplex<f32>>
# CHECK-SAME: tensor<7xf32>
print_ir(np.empty((7, ), np.complex64))(partial(lax.convert_element_type,
new_dtype=np.float32))
# CHECK-LABEL: TEST: convert_element_type float32[7]
# CHECK: mhlo.compare
# CHECK-SAME: tensor<7xf32>
# CHECK-SAME: tensor<7xi1>
print_ir(np.empty((7, ), np.float32))(partial(lax.convert_element_type,
new_dtype=np.bool_))
# CHECK-LABEL: TEST: clz uint32[]
# CHECK: mhlo.count_leading_zeros
# CHECK-SAME: tensor<ui32>
print_ir(np.uint32(0))(lax.clz)
# CHECK-LABEL: TEST: conj complex64[]
# CHECK-DAG: mhlo.real
# CHECK-DAG: mhlo.imag
# CHECK-DAG: mhlo.neg
# CHECK-DAG: mhlo.complex
# CHECK-SAME: tensor<complex<f32>>
print_ir(np.complex64(0))(lax.conj)
# CHECK-LABEL: TEST: cos float32[]
# CHECK: mhlo.cos
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.cos)
# CHECK-LABEL: TEST: cosh float32[]
# CHECK: chlo.cosh
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.cosh)
# CHECK-LABEL: TEST: digamma float32[]
# CHECK: chlo.digamma
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.digamma)
# CHECK-LABEL: TEST: div float32[] float32[]
# CHECK: mhlo.div
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1), np.float32(2))(lax.div)
# CHECK-LABEL: TEST: eq float32[] float32[]
# CHECK: mhlo.compare
# CHECK-SAME: compare_type = #mhlo<"comparison_type FLOAT">
# CHECK-SAME: comparison_direction = #mhlo<"comparison_direction EQ">
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1), np.float32(2))(lax.eq)
# CHECK-LABEL: TEST: eq complex128[] complex128[]
# CHECK: mhlo.compare
# CHECK-SAME: compare_type = #mhlo<"comparison_type FLOAT">
# CHECK-SAME: comparison_direction = #mhlo<"comparison_direction EQ">
# CHECK-SAME: tensor<complex<f64>>
print_ir(np.complex128(1), np.complex128(2))(lax.eq)
# CHECK-LABEL: TEST: eq int64[] int64[]
# CHECK: mhlo.compare
# CHECK-SAME: compare_type = #mhlo<"comparison_type SIGNED">
# CHECK-SAME: comparison_direction = #mhlo<"comparison_direction EQ">
# CHECK-SAME: tensor<i64>
print_ir(np.int64(1), np.int64(2))(lax.eq)
# CHECK-LABEL: TEST: eq uint16[] uint16[]
# CHECK: mhlo.compare
# CHECK-SAME: compare_type = #mhlo<"comparison_type UNSIGNED">
# CHECK-SAME: comparison_direction = #mhlo<"comparison_direction EQ">
# CHECK-SAME: tensor<ui16>
print_ir(np.uint16(1), np.uint16(2))(lax.eq)
# CHECK-LABEL: TEST: erf float32[]
# CHECK: chlo.erf
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.erf)
# CHECK-LABEL: TEST: erfc float32[]
# CHECK: chlo.erfc
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.erfc)
# CHECK-LABEL: TEST: erf_inv float32[]
# CHECK: xla_fallback_erf_inv
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.erf_inv)
# CHECK-LABEL: TEST: exp float16[]
# CHECK: mhlo.exp
# CHECK-SAME: tensor<f16>
print_ir(np.float16(0))(lax.exp)
# CHECK-LABEL: TEST: expm1 bfloat16[]
# CHECK: mhlo.exponential_minus_one
# CHECK-SAME: tensor<bf16>
print_ir(jnp.bfloat16(0))(lax.expm1)
# CHECK-LABEL: TEST: floor bfloat16[2,3]
# CHECK: mhlo.floor
# CHECK-SAME: tensor<2x3xbf16>
print_ir(np.empty((2, 3), jnp.bfloat16))(lax.floor)
# CHECK-LABEL: TEST: ge float32[] float32[]
# CHECK: mhlo.compare
# CHECK-SAME: compare_type = #mhlo<"comparison_type FLOAT">
# CHECK-SAME: comparison_direction = #mhlo<"comparison_direction GE">
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1), np.float32(2))(lax.ge)
# CHECK-LABEL: TEST: gt float32[] float32[]
# CHECK: mhlo.compare
# CHECK-SAME: compare_type = #mhlo<"comparison_type FLOAT">
# CHECK-SAME: comparison_direction = #mhlo<"comparison_direction GT">
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1), np.float32(2))(lax.gt)
# CHECK-LABEL: TEST: igamma float32[] float32[]
# CHECK: xla_fallback_igamma
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0), np.float32(0))(lax.igamma)
# CHECK-LABEL: TEST: igammac float32[] float32[]
# CHECK: xla_fallback_igammac
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0), np.float32(0))(lax.igammac)
# CHECK-LABEL: TEST: igamma_grad_a float32[] float32[]
# CHECK: xla_fallback_igamma_grad_a
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0), np.float32(0))(lax.igamma_grad_a)
# CHECK-LABEL: TEST: imag complex64[]
# CHECK: mhlo.imag
# CHECK-SAME: tensor<complex<f32>>
print_ir(np.complex64(0))(lax.imag)
# CHECK-LABEL: TEST: integer_pow float32[]
# CHECK-DAG: mhlo.mul
# CHECK-SAME: tensor<f32>
@print_ir(np.float32(1))
def integer_pow(x):
return lax.integer_pow(x, 3)
# CHECK-LABEL: TEST: is_finite float64[]
# CHECK: mhlo.is_finite
# CHECK-SAME: tensor<f64>
print_ir(np.float64(0))(lax.is_finite)
# CHECK-LABEL: TEST: le float32[] float32[]
# CHECK: mhlo.compare
# CHECK-SAME: compare_type = #mhlo<"comparison_type FLOAT">
# CHECK-SAME: comparison_direction = #mhlo<"comparison_direction LE">
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1), np.float32(2))(lax.le)
# CHECK-LABEL: TEST: lgamma float32[]
# CHECK: chlo.lgamma
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.lgamma)
# CHECK-LABEL: TEST: log float32[]
# CHECK: mhlo.log
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.log)
# CHECK-LABEL: TEST: log1p float32[]
# CHECK: mhlo.log_plus_one
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.log1p)
# CHECK-LABEL: TEST: lt float32[] float32[]
# CHECK: mhlo.compare
# CHECK-SAME: compare_type = #mhlo<"comparison_type FLOAT">
# CHECK-SAME: comparison_direction = #mhlo<"comparison_direction LT">
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1), np.float32(2))(lax.lt)
# CHECK-LABEL: TEST: max float32[] float32[]
# CHECK: mhlo.max
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1), np.float32(2))(lax.max)
# CHECK-LABEL: TEST: min float32[] float32[]
# CHECK: mhlo.min
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1), np.float32(2))(lax.min)
# CHECK-LABEL: TEST: mul float32[] float32[]
# CHECK: mhlo.mul
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1), np.float32(2))(lax.mul)
# CHECK-LABEL: TEST: ne float32[] float32[]
# CHECK: mhlo.compare
# CHECK-SAME: compare_type = #mhlo<"comparison_type FLOAT">
# CHECK-SAME: comparison_direction = #mhlo<"comparison_direction NE">
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1), np.float32(2))(lax.ne)
# CHECK-LABEL: TEST: neg int64[]
# CHECK: mhlo.negate
# CHECK-SAME: tensor<i64>
print_ir(np.int64(0))(lax.neg)
# CHECK-LABEL: TEST: nextafter float32[] float32[]
# CHECK: chlo.next_after
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0), np.float32(0))(lax.nextafter)
# CHECK-LABEL: TEST: bitwise_not int64[]
# CHECK: mhlo.not
# CHECK-SAME: tensor<i64>
print_ir(np.int64(0))(lax.bitwise_not)
# CHECK-LABEL: TEST: bitwise_not bool[]
# CHECK: mhlo.not
# CHECK-SAME: tensor<i1>
print_ir(np.bool_(0))(lax.bitwise_not)
# CHECK-LABEL: TEST: population_count uint32[]
# CHECK: mhlo.popcnt
# CHECK-SAME: tensor<ui32>
print_ir(np.uint32(0))(lax.population_count)
# CHECK-LABEL: TEST: pow float32[] float32[]
# CHECK: mhlo.power
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1), np.float32(2))(lax.pow)
# CHECK-LABEL: TEST: random_gamma_grad float32[] float32[]
# CHECK: xla_fallback_random_gamma_grad
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0), np.float32(0))(lax.random_gamma_grad)
# CHECK-LABEL: TEST: real complex128[]
# CHECK: mhlo.real
# CHECK-SAME: tensor<complex<f64>>
print_ir(np.complex128(0))(lax.real)
# CHECK-LABEL: TEST: reduce_precision bfloat16[]
# CHECK: mhlo.reduce_precision
# CHECK-SAME: tensor<bf16>
print_ir(jnp.bfloat16(0))(partial(lax.reduce_precision,
exponent_bits=2,
mantissa_bits=2))
# CHECK-LABEL: TEST: rem float32[] float32[]
# CHECK: mhlo.rem
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1), np.float32(2))(lax.rem)
# CHECK-LABEL: TEST: round float64[7,1]
# CHECK: mhlo.round
# CHECK-SAME: tensor<7x1xf64>
print_ir(np.empty((7, 1), np.float64))(partial(
lax.round, rounding_method=lax.RoundingMethod.AWAY_FROM_ZERO))
# CHECK-LABEL: TEST: rsqrt complex64[]
# CHECK: mhlo.rsqrt
# CHECK-SAME: tensor<complex<f32>>
print_ir(jnp.complex64(0))(lax.rsqrt)
# CHECK-LABEL: TEST: shift_left uint32[] uint32[]
# CHECK: mhlo.shift_left
# CHECK-SAME: tensor<ui32>
print_ir(np.uint32(0), np.uint32(0))(lax.shift_left)
# CHECK-LABEL: TEST: shift_right_arithmetic uint8[] uint8[]
# CHECK: mhlo.shift_right_arithmetic
# CHECK-SAME: tensor<ui8>
print_ir(np.uint8(0), np.uint8(0))(lax.shift_right_arithmetic)
# CHECK-LABEL: TEST: shift_right_logical uint16[] uint16[]
# CHECK: mhlo.shift_right_logical
# CHECK-SAME: tensor<ui16>
print_ir(np.uint16(0), np.uint16(0))(lax.shift_right_logical)
# CHECK-LABEL: TEST: sign int64[]
# CHECK: mhlo.sign
# CHECK-SAME: tensor<i64>
print_ir(np.int64(0))(lax.sign)
# CHECK-LABEL: TEST: sign uint32[]
# CHECK: mhlo.compare
# CHECK-SAME: tensor<ui32>
print_ir(np.uint32(0))(lax.sign)
# CHECK-LABEL: TEST: sin float32[]
# CHECK: mhlo.sin
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.sin)
# CHECK-LABEL: TEST: sinh float32[]
# CHECK: chlo.sinh
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.sinh)
# CHECK-LABEL: TEST: sub float32[] float32[]
# CHECK: mhlo.sub
# CHECK-SAME: tensor<f32>
print_ir(np.float32(1), np.float32(2))(lax.sub)
# CHECK-LABEL: TEST: sqrt bfloat16[]
# CHECK: mhlo.sqrt
# CHECK-SAME: tensor<bf16>
print_ir(jnp.bfloat16(0))(lax.sqrt)
# CHECK-LABEL: TEST: tan float16[]
# CHECK: chlo.tan
# CHECK-SAME: tensor<f16>
print_ir(np.float16(0))(lax.tan)
# CHECK-LABEL: TEST: tanh float32[]
# CHECK: mhlo.tanh
# CHECK-SAME: tensor<f32>
print_ir(np.float32(0))(lax.tanh)
def fkt(a):
return np.bool_(a)
def test_numpy_bool_(self):
self.assertReceivedEqualsSent(numpy.bool_(True), True)
self.assertReceivedEqualsSent(numpy.bool_(False), False)
class Board:
"""
Board manages the game state
It handles all transformations from one valid game state to another
"""
costs = {
"Road": {
"Wood": 1,
"Brick": 1,
"Wheat": 0,
"Rock": 0,
"Sheep": 0
},
"Settlement": {
"Wood": 1,
"Brick": 1,
"Wheat": 1,
"Rock": 0,
"Sheep": 1
},
"City": {
"Wood": 0,
"Brick": 0,
"Wheat": 2,
"Rock": 3,
"Sheep": 0
},
"Development Card": {
"Wood": 0,
"Brick": 0,
"Wheat": 1,
"Rock": 1,
"Sheep": 1
}
}
# cols are junctions, rows are roads
road_adjacency = pd.read_csv("road.csv", header=0, index_col=0)
road_adjacency = road_adjacency.apply(lambda x: np.bool_(x))
# cols are junctions, rows are tiles
tile_adjacency = pd.read_csv("tile.csv", header=0, index_col=0)
tile_adjacency = tile_adjacency.apply(lambda x: np.bool_(x))
@staticmethod
def s2r(location):
x = Board.road_adjacency.iloc[:, location]
return x
@staticmethod
def r2s(location):
x = Board.road_adjacency.iloc[location, :]
return x
@staticmethod
def r2r(location):
s = Board.r2s(location)
r = np.any(Board.road_adjacency.loc[:, s], 1)
r[location] = False
return r
@staticmethod
def s2s(location):
r = Board.s2r(location)
s = np.any(Board.road_adjacency.loc[r, :], 0)
s[location] = False
return s
@staticmethod
def s2t(location):
return Board.tile_adjacency.iloc[:, location]
@staticmethod
def t2s(location):
return Board.tile_adjacency.iloc[location, :]
def __init__(self,
manager,
n_players,
border_setup="standard",
tile_setup="random"):
"""
This sets up everything prior to initial settlement placement.
Here's what it does:
Assigns resources to each tile, including robbing the Desert
Assigns numbers to the tiles based on the predefined order
Generates the port arrangement
Shuffles the Dev card deck
Initializes all roads and cities to 0
Sets the turn counter and player counters to 0
"""
self.manager = manager
resources = [
"Brick", "Wheat", "Sheep", "Sheep", "Rock", "Brick", "Wood",
"Sheep", "Rock", "Wheat", "Wheat", "Wood", "Rock", "Wheat", "Wood",
"Sheep", "Brick", "Wood", "Desert"
]
if tile_setup == "random":
random.shuffle(resources)
elif tile_setup == "basic":
pass
else:
raise ValueError("Bad Tile Setup")
number_order = [
5, 6, 11, 5, 8, 10, 9, 2, 10, 12, 9, 8, 3, 4, 3, 4, 6, 11
]
self.tiles = []
for resource in resources:
if resource == "Desert":
self.tiles.append(Tile("Desert", 0))
else:
self.tiles.append(Tile(resource, number_order.pop(0)))
self.border = Border(border_setup)
self.roads = np.zeros(72, dtype=np.int8)
self.settlements = np.zeros(54, dtype=np.int8)
self.cities = np.zeros(54, dtype=np.int8)
self.devdeck = [
*["Knight"] * 14, *["Monopoly"] * 2, *["Road Building"] * 2,
*["Victory"] * 5, *["Year of Plenty"] * 2
]
random.shuffle(self.devdeck)
# Using an extra player
# to make players array practically start at 0
self.players = [Player(self) for x in range(n_players + 1)]
self.last_turn = 0
self.turn_number = 0
def check_road_length(self, player, location):
"""
This is the most computer-science intensive part of the game
It crawls through the road-adjacency graph recursively
to find the longest road
"""
def crawler(used, location, mode="distance"):
joints = Board.road_adjacency.iloc[:, location]
if sum(joints) != 2:
print("Bad adjacency graph: road doesn't have 2 endpoints")
options = np.zeros(72, dtype=np.int8)
for i, joint in enumerate(joints):
options += self.roads * \
Board.road_adjacency.iloc[joint, :] == player
options = options * (1 - used)
here = np.zeros(72, dtype=np.int8)
if sum(options) > 0:
paths = []
for i, option in enumerate(options):
if option == 1:
here = np.zeros(72, dtype=np.int8)
here[location] = 1
paths.append(crawler(used + here, i))
if mode == "distance":
return max(paths) + 1
elif mode == "ends":
return np.sum(paths, 0)
return False
else:
if mode == "distance":
return 1
elif mode == "ends":
return here
return False
right_points = Board.road_adjacency.iloc[:, location]
for i, h in enumerate(right_points):
if h == 1:
right_points[h] = 0
left_endpoints = crawler([right_points], location, mode="ends")
longest_road = 0
for i, endpoint in enumerate(left_endpoints):
if endpoint == 1:
longest_road = max(longest_road, crawler([],
i,
mode="distance"))
print("Found a long road:", longest_road)
if longest_road >= 5:
if self.players[player].road_length < longest_road:
self.players[player].road_length = longest_road
for other_player in self.players:
if other_player.road_length >= longest_road:
break
else:
for other_player in self.players:
player.longest_road = False
self.players[player].longest_road = True
def build_road(self, player, location, free=False):
if self.roads[location] == 0 and self.players[player].roads > 0:
next_settlements = Board.r2s(location)
next_roads = Board.r2r(location)
has_settlement = sum(self.settlements[next_settlements] == player)
has_city = sum(self.cities[next_settlements] == player)
has_road = sum(self.roads[next_roads] == player)
if has_settlement + has_road + has_city > 0:
if free or self.players[player].spend(Board.costs["Road"]):
self.roads[location] = player
self.players[player].roads -= 1
self.check_road_length(player, location)
return True
print("No Resources!")
return False
print("No connection!")
return False
print("You're Blocked!")
return False
def build_settlement(self, player, location, initial=False, getnear=False):
blocking_locations = Board.s2s(location)
if sum((self.settlements + self.cities) * blocking_locations) == 0:
if sum(self.roads[Board.s2r(location)] == player) > 0:
if self.players[player].spend(Board.costs["Settlement"]):
self.settlements[location] = player
self.players[player].addvp()
self.players[player].settlements -= 1
self.players[player].addport(self.border.port(location))
return True
else:
print("No Resources!")
elif initial == True:
self.settlements[location] = player
self.players[player].settlements -= 1
self.players[player].addvp()
self.players[player].addport(self.border.port(location))
if getnear:
for i, t in enumerate(Board.s2t(location)):
if t:
self.players[player].get(self.tiles[i].give())
return True
else:
print("No Road")
else:
print("You're Blocked")
return False
def build_city(self, player, location):
if self.settlements[
location] == player and self.players[player].cities > 0:
if self.players[player].spend(Board.costs["City"]):
self.settlements[location] = 0
self.players[player].cities -= 1
self.players[player].settlements += 1
self.cities[location] = player
return True
print("No Resources!")
return False
print("You need a settlement there first!")
return False
def buy_devcard(self, player):
if len(self.devdeck) > 0:
if self.players[player].spend(Board.costs["Development Card"]):
self.players[player].get_devcard(self.devdeck.pop())
return True
print("No Resources!")
return False
print("You losers used all the dev cards!")
return False
def roll(self, fix=None):
if self.last_turn == len(self.players):
self.last_turn = 1
self.turn_number += 1
else:
self.last_turn += 1
result = random.randint(1, 6) + random.randint(1, 6)
if fix is not None:
result = fix
if result != 7:
for i, tile in enumerate(self.tiles):
if tile.produce(result):
spots = Board.t2s(i)
settlements = self.settlements[spots]
cities = self.cities[spots]
for settlement in settlements:
if settlement != 0:
self.players[settlement].get(tile.give(1))
for city in cities:
if city != 0:
self.players[city].get(tile.give(2))
return result
else:
return 7
def rob(self, player1, player2, location):
if self.tiles[location].robber == False:
spots = Board.t2s(location)
if sum(self.settlements[spots] == player2) > 0 or sum(
self.cities[spots] == player2) > 0:
for tile in self.tiles:
tile.clearrobber()
self.tiles[location].rob()
self.players[player1].get(self.players[player2].take_random())
return True
print("That player isn't next to that tile!")
return False
print("The robber must move to a different tile!")
return False
def knight(self, player, special):
if self.rob(player, special[0], special[1]):
self.players[player].knight_count += 1
# Check for largest army
if self.players[player].knight_count > 2:
for i, _ in enumerate(self.players):
if i != player:
if self.players[i].knight_count >= self.players[
player].knight_count:
break
else: # If the loop didn't break
for p in self.players:
if p.largest_army:
p.largest_army = False
p.victory_points -= 2
self.players[player].largest_army = True
self.players[player].addvp()
self.players[player].addvp()
return True
return False
def monopoly(self, player, special):
if type(special) == str and special in bank_statement.keys():
for i, player in enumerate(self.players):
receipt = bank_statement.copy()
resource_number = player.resources[special]
receipt[special] = resource_number
self.players[player].get(self.players[i].spend(receipt))
return True
print("Monopoly takes a bank statement!")
return False
def year_of_plenty(self, player, special):
if type(special) == dict and len(special) == 5:
if sum(special.values()) == 2:
self.players[player].get(special)
return True
print("That wasn't two resources!")
return False
print("That wasn't a receipt!")
return False
def road_building(self, player, special):
if len(special) == 2 and type(special[0]) == int and type(
special[1]) == int:
if self.build_road(player, special[0],
free=True) and self.build_road(
player, special[1], free=True):
return True
print("That isn't 2 locations!")
return False
def activate_devcard(self, player, card, turn_number, special):
if self.players[player].can_flip_devcard(card, self.turn_number):
if card == "Knight":
if self.knight(player, special):
self.players[player].flip_devcard(card, turn_number)
return True
elif card == "Victory":
self.players[player].addvp()
self.players[player].flip_devcard(card, turn_number)
return True
elif card == "Monopoly":
if self.monopoly(player, special):
self.players[player].flip_devcard(card, turn_number)
return True
elif card == "Road Building":
if self.road_building(player, special):
self.players[player].flip_devcard(card, turn_number)
return True
return False
elif card == "Year of Plenty":
if self.year_of_plenty(player, special):
self.players[player].flip_devcard(card, turn_number)
return True
else:
print("Not a valid card")
return False
print("You can't use that dev card!")
return False
def test_numpy_scalar_conversion_values(self):
self.assertEqual(nd.as_py(nd.array(np.bool_(True))), True)
self.assertEqual(nd.as_py(nd.array(np.bool_(False))), False)
self.assertEqual(nd.as_py(nd.array(np.int8(100))), 100)
self.assertEqual(nd.as_py(nd.array(np.int8(-100))), -100)
self.assertEqual(nd.as_py(nd.array(np.int16(20000))), 20000)
self.assertEqual(nd.as_py(nd.array(np.int16(-20000))), -20000)
self.assertEqual(nd.as_py(nd.array(np.int32(1000000000))), 1000000000)
self.assertEqual(nd.as_py(nd.array(np.int64(-1000000000000))),
-1000000000000)
self.assertEqual(nd.as_py(nd.array(np.int64(1000000000000))),
1000000000000)
self.assertEqual(nd.as_py(nd.array(np.int32(-1000000000))),
-1000000000)
self.assertEqual(nd.as_py(nd.array(np.uint8(200))), 200)
self.assertEqual(nd.as_py(nd.array(np.uint16(50000))), 50000)
self.assertEqual(nd.as_py(nd.array(np.uint32(3000000000))), 3000000000)
self.assertEqual(nd.as_py(nd.array(np.uint64(10000000000000000000))),
10000000000000000000)
self.assertEqual(nd.as_py(nd.array(np.float32(2.5))), 2.5)
self.assertEqual(nd.as_py(nd.array(np.float64(2.5))), 2.5)
self.assertEqual(nd.as_py(nd.array(np.complex64(2.5 - 1j))), 2.5 - 1j)
self.assertEqual(nd.as_py(nd.array(np.complex128(2.5 - 1j))), 2.5 - 1j)
if np.__version__ >= '1.7':
# Various date units
self.assertEqual(nd.as_py(nd.array(np.datetime64('2000'))),
date(2000, 1, 1))
self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12'))),
date(2000, 12, 1))
self.assertEqual(nd.as_py(nd.array(np.datetime64('2000-12-13'))),
date(2000, 12, 13))
# Various datetime units
self.assertEqual(
nd.as_py(nd.array(np.datetime64('2000-12-13T12Z'))),
datetime(2000, 12, 13, 12, 0))
self.assertEqual(
nd.as_py(nd.array(np.datetime64('2000-12-13T12:30Z'))),
datetime(2000, 12, 13, 12, 30))
self.assertEqual(
nd.as_py(nd.array(np.datetime64('1823-12-13T12:30Z'))),
datetime(1823, 12, 13, 12, 30))
self.assertEqual(
nd.as_py(nd.array(np.datetime64('2000-12-13T12:30:24Z'))),
datetime(2000, 12, 13, 12, 30, 24))
self.assertEqual(
nd.as_py(nd.array(np.datetime64('2000-12-13T12:30:24.123Z'))),
datetime(2000, 12, 13, 12, 30, 24, 123000))
self.assertEqual(
nd.as_py(nd.array(
np.datetime64('2000-12-13T12:30:24.123456Z'))),
datetime(2000, 12, 13, 12, 30, 24, 123456))
self.assertEqual(
nd.as_py(
nd.array(np.datetime64('2000-12-13T12:30:24.123456124Z'))),
datetime(2000, 12, 13, 12, 30, 24, 123456))
self.assertEqual(
str(nd.array(np.datetime64('2000-12-13T12:30:24.123456124Z'))),
'2000-12-13T12:30:24.1234561Z')
self.assertEqual(
str(nd.array(np.datetime64('1842-12-13T12:30:24.123456124Z'))),
'1842-12-13T12:30:24.1234561Z')
def convert_bool(self, v, t):
"""Convert the scalar to a TVM array."""
return tvm.runtime.ndarray.array(np.bool_(v), self.context)
def test_scalar_array_bool(self):
# Numpy bools can be used as boolean index (python ones as of yet not)
a = np.array(1)
assert_equal(a[np.bool_(True)], a[np.array(True)])
assert_equal(a[np.bool_(False)], a[np.array(False)])
def setup(self):
self.int64 = np.int64(1)
self.integer_array = np.array([[1, 2], [-1, 2]])
self.boolean_array = np.array([[True, False], [False, False]])
self.bool_ = np.bool_(False)
self.tuple = (slice(0, 4, 2), ..., 1)
def stream_network(Z, flats, sills, interiorbasins, cellsize):
#[Iobj,SILLS,IntBasin] = identifyflats(Z);
Z = Z.data
Z_ravel = np.ravel(Z)
nrc = Z_ravel.shape[0]
Iobj = flats
SILLS = sills
IntBasin = interiorbasins
# Here we choose the distance transform from outside the lakes to the inside and take the locations as sills where the distance is maximum.
DD = scipy.ndimage.distance_transform_edt(np.bitwise_not(IntBasin))
MaxIntIX = [0, 0] #added to prevent MaxIntIX does not exist errors
IntBasin_labels = skimage.measure.label(IntBasin)
for r in np.arange(1, np.max(IntBasin_labels)):
PixelIdxList = np.argwhere(IntBasin_labels == r)
ixm = np.argmax(DD[IntBasin_labels == r])
MaxIntIX = PixelIdxList[ixm]
Iobj[PixelIdxList[0][0], PixelIdxList[0][1]] = 0
SILLS[PixelIdxList[0][0], PixelIdxList[0][1]] = 1
ixm = MaxIntIX
Iobj[ixm[0], ixm[1]] = 0
SILLS[ixm[0], ixm[1]] = 1
# establish the connectivity between sills and flats
#dem = ZintoDB;
whereSILLS = np.argwhere(SILLS)
rows = []
cols = []
for rowcol in whereSILLS:
[row, col] = rowcol
rows = np.append(rows, row)
cols = np.append(cols, col)
IXsill = [rows, cols]
rowadd = [-1, -1, 0, 1, 1, 1, 0, -1]
coladd = [0, 1, 1, 1, 0, -1, -1, -1]
PreSillPixel = [0]
for r in np.arange(0, 8):
rowp = rows + rowadd[r]
colp = cols + coladd[r]
ValidRowColPair1 = np.bitwise_and(rowp > 0, colp > 0)
ValidRowColPair2 = np.bitwise_and(rowp < Z.shape[0], colp < Z.shape[1])
ValidRowColPair = np.bitwise_and(ValidRowColPair1, ValidRowColPair2)
whereValidRowColPair = np.where(ValidRowColPair)
IXPreSill = [rowp[whereValidRowColPair], colp[whereValidRowColPair]]
I1 = np.ravel_multi_index([
np.int_(rows[whereValidRowColPair]),
np.int_(cols[whereValidRowColPair])
], Z.shape)
I2 = np.ravel_multi_index(
[np.int_(IXPreSill[0]),
np.int_(IXPreSill[1])], Z.shape)
I3 = np.ravel_multi_index(
[np.int_(IXPreSill[0]),
np.int_(IXPreSill[1])], Z.shape)
#PreSillPixelCondition = (np.argwhere(np.bitwise_and((Z_ravel[I1] ==
# Z_ravel[I2]),
# Iobj.ravel()[I3]))
# if np.count_nonzero(PreSillPixelCondition)>0:
# for i in np.arange(0,len(PreSillPixelCondition)):
# PreSillPixelAddition = ([IXPreSill[0][PreSillPixelCondition[i]],IXPreSill[1][PreSillPixelCondition[i]]])
# PreSillPixel.append(PreSillPixelAddition)
# else:
# continue
PreSillPixel.pop(0)
Iobj = np.bitwise_not(Iobj)
D = scipy.ndimage.distance_transform_edt(Iobj)
masked = np.inf * np.ones(Z.shape, D.dtype)
masked[Iobj] = 0
D[np.bitwise_not(Iobj)] = np.inf
D = ((skimage.morphology.reconstruction(
seed=D + 1, mask=masked, method='erosion')) - D) * cellsize
D = np.nan_to_num(D)
D[Iobj] = 0
#D = D**-1
cost_field = map_image_to_costs(D**-1, PreSillPixel)
D = _wdt_python(cost_field) + 1
D[Iobj] = -np.inf
#del PreSillPixel
V = np.reshape(D.data, [1, D.shape[0] * D.shape[1]])
if np.any(np.isnan(np.diff(D.ravel()))):
IXSortedFlats = np.arange(0, len(Z_ravel))
IXSortedFlats = IXSortedFlats[::-1]
else:
IXSortedFlats = np.argsort(D.ravel())
IXSortedFlats = IXSortedFlats[::-1]
#del D
ndx = np.arange(np.uint32(0), np.uint32(nrc))
ndx = ndx[IXSortedFlats]
ndx = np.arange(np.uint32(0), np.uint32(nrc))
ndx = ndx[IXSortedFlats]
del IXSortedFlats
ix = np.argsort(Z_ravel[ndx])
ix = ix[::-1]
ix = ndx[ix]
del ndx
# a fast solution that has quite much memory overhead...
pp = np.zeros(Z_ravel.shape, dtype=np.int32)
IX = np.arange(np.int32(0), np.int32(Z_ravel.shape))
pp[ix] = IX
pp = pp.reshape(Z.shape)
# cardinal neighbors
IXC1 = skimage.morphology.dilation(pp, skimage.morphology.selem.diamond(1))
IXC1 = IXC1.ravel()
xxx1 = IXC1
IX = IXC1[ix]
IXC1 = ix[IX]
G1 = (Z_ravel[ix] - Z_ravel[IXC1]) / (cellsize)
I4 = (np.argwhere(ix == IXC1)).ravel()
I4 = list(I4)
I4_test = np.zeros(G1.shape)
I4_test[I4] = -np.inf
G1 = G1 + I4_test
#G1[ix == IXC1] = -np.inf;
# diagonal neighbors
kernel = np.array([[1, 0, 1], [0, 1, 0], [1, 0, 1]])
IXC2 = skimage.morphology.dilation(pp, kernel)
IXC2 = IXC2.ravel()
xxx2 = IXC2
IX = IXC2[ix]
IXC2 = ix[IX]
G2 = (Z_ravel[ix] - Z_ravel[IXC2]) / np.linalg.norm([cellsize, cellsize])
# choose the steeper one
#I = np.bitwise_and(G1<=G2, xxx2[ix]>xxx1[ix]);
I = dask.array.bitwise_and(dask.array.less_equal(G1, G2),
xxx2[ix] > xxx1[ix])
ixc = IXC1
ixc[I] = IXC2[I]
I = ixc == ix
ix = ix[np.bitwise_not(I)]
ixc = ixc[np.bitwise_not(I)]
# remove nans
I = np.isnan(Z_ravel)
ixc = ixc[~I[ix]]
ix = ix[~I[ix]]
ix = np.int_(ix[~np.isnan(ix)])
ixc = np.int_(ixc[~np.isnan(ixc)])
A = np.ones(Z.shape)
for r in np.arange(0, len(ix)):
[ixcx, ixcy] = np.unravel_index(ixc[r], Z.shape)
[ixx, ixy] = np.unravel_index(ix[r], Z.shape)
A[ixcx, ixcy] = A[ixx, ixy] + A[ixcx, ixcy]
DBcounter = 0
D = np.zeros(Z_ravel.shape[0], dtype=np.int32)
outlets = np.zeros((len(ix), 1))
for r in np.arange(len(ix) - 1, 1, -1):
if D[ixc[r]] == 0:
DBcounter = DBcounter + 1
D[ixc[r]] = DBcounter
outlets[DBcounter] = ixc[r]
D[ix[r]] = D[ixc[r]]
D = D.reshape(Z.shape)
minarea = 1000
W = np.bitwise_and(np.bool_(D > 0), np.bool_(A > minarea))
Z = np.zeros(W.shape)
[ixx, ixy] = np.unravel_index(ix, Z.shape)
Z[ixx, ixy] = W[ixx, ixy]
I = np.argwhere(Z[ixx, ixy])
[ixcx, ixcy] = np.unravel_index(ix, Z.shape)
Z[ixcx[I], ixcy[I]] = W[ixcx[I], ixcy[I]]
W = Z * cellsize
return W
np.timedelta64()
np.timedelta64(0)
np.timedelta64(0, "D")
np.timedelta64(0, ('ms', 3))
np.timedelta64(0, b"D")
np.timedelta64("3")
np.timedelta64(b"5")
np.timedelta64(np.timedelta64(2))
np.timedelta64(dt.timedelta(2))
np.timedelta64(None)
np.timedelta64(None, "D")
np.void(1)
np.void(np.int64(1))
np.void(True)
np.void(np.bool_(True))
np.void(b"test")
np.void(np.bytes_("test"))
# Protocols
i8 = np.int64()
u8 = np.uint64()
f8 = np.float64()
c16 = np.complex128()
b_ = np.bool_()
td = np.timedelta64()
U = np.str_("1")
S = np.bytes_("1")
AR = np.array(1, dtype=np.float64)
int(i8)
elif typ is pd.CategoricalIndex:
element = idx.categories[0]
return pd.CategoricalIndex([element, element],
categories=idx.categories,
ordered=idx.ordered,
name=idx.name)
elif typ is pd.MultiIndex:
levels = [_nonempty_index(i) for i in idx.levels]
labels = [[0, 0] for i in idx.levels]
return pd.MultiIndex(levels=levels, labels=labels, names=idx.names)
raise TypeError("Don't know how to handle index of "
"type {0}".format(type(idx).__name__))
_simple_fake_mapping = {
'b': np.bool_(True),
'V': np.void(b' '),
'M': np.datetime64('1970-01-01'),
'm': np.timedelta64(1),
'S': np.str_('foo'),
'a': np.str_('foo'),
'U': np.unicode_('foo'),
'O': 'foo'
}
def _scalar_from_dtype(dtype):
if dtype.kind in ('i', 'f', 'u'):
return dtype.type(1)
elif dtype.kind == 'c':
return dtype.type(complex(1, 0))
def test_numpybool_(self):
nptrue = np.bool_(True)
self.assertEqual(self.encoder.default(nptrue), True)
self.assertIsInstance(self.encoder.default(nptrue), bool)
def test_db2(name):
if name == 'postgresql':
pytest.importorskip('psycopg2')
if os.environ.get('POSTGRES_DB'): # gitlab-ci
name = 'postgresql://*****:*****@postgres:5432/testase'
else:
name = os.environ.get('ASE_TEST_POSTGRES_URL')
if name is None:
return
elif name == 'mysql':
pytest.importorskip('pymysql')
if os.environ.get('CI_PROJECT_DIR'): # gitlab-ci
name = 'mysql://*****:*****@mysql:3306/testase_mysql'
else:
name = os.environ.get('MYSQL_DB_URL')
if name is None:
return
elif name == 'mariadb':
pytest.importorskip('pymysql')
if os.environ.get('CI_PROJECT_DIR'): # gitlab-ci
name = 'mariadb://*****:*****@mariadb:3306/testase_mysql'
else:
name = os.environ.get('MYSQL_DB_URL')
if name is None:
return
c = connect(name)
print(name, c)
if 'postgres' in name or 'mysql' in name or 'mariadb' in name:
c.delete([row.id for row in c.select()])
id = c.reserve(abc=7)
c.delete([d.id for d in c.select(abc=7)])
id = c.reserve(abc=7)
assert c[id].abc == 7
a = c.get_atoms(id)
c.write(Atoms())
ch4 = molecule('CH4', calculator=EMT())
ch4.constraints = [FixAtoms(indices=[1]), FixBondLength(0, 2)]
f1 = ch4.get_forces()
print(f1)
c.delete([d.id for d in c.select(C=1)])
chi = np.array([1 + 0.5j, 0.5])
id = c.write(ch4, data={'1-butyne': 'bla-bla', 'chi': chi})
row = c.get(id)
print(row.data['1-butyne'], row.data.chi)
assert (row.data.chi == chi).all(), (row.data.chi, chi)
print(row)
assert len(c.get_atoms(C=1).constraints) == 2
f2 = c.get(C=1).forces
assert abs(f2.sum(0)).max() < 1e-14
f3 = c.get_atoms(C=1).get_forces()
assert abs(f1 - f3).max() < 1e-14
a = read(name, index='id={}'.format(id))[0]
f4 = a.get_forces()
assert abs(f1 - f4).max() < 1e-14
with must_raise(ValueError):
c.update(id, abc={'a': 42})
c.update(id, grr='hmm')
row = c.get(C=1)
assert row.id == id
assert (row.data.chi == chi).all()
for row in c.select(include_data=False):
assert len(row.data) == 0
with must_raise(ValueError):
c.write(ch4, foo=['bar', 2]) # not int, bool, float or str
with must_raise(ValueError):
c.write(Atoms(), pi='3.14') # number as a string
with must_raise(ValueError):
c.write(Atoms(), fmax=0.0) # reserved word
with must_raise(ValueError):
c.write(Atoms(), S=42) # chemical symbol as key
id = c.write(Atoms(),
b=np.bool_(True),
i=np.int64(42),
n=np.nan,
x=np.inf,
s='NaN2',
A=42)
row = c[id]
assert isinstance(row.b, bool)
assert isinstance(row.i, int)
assert np.isnan(row.n)
assert np.isinf(row.x)
# Make sure deleting a single key works:
id = c.write(Atoms(), key=7)
c.update(id, delete_keys=['key'])
assert 'key' not in c[id]
e = [row.get('energy') for row in c.select(sort='energy')]
assert len(e) == 5 and abs(e[0] - 1.991) < 0.0005
# Test the offset keyword
ids = [row.get('id') for row in c.select()]
offset = 2
assert next(c.select(offset=offset)).id == ids[offset]
def similarity_measure_euclidean(ds_tar, ds_src, results, p_value):
print 'Computing Euclidean similarity...'
#classifier = results['classifier']
# Get classifier from results
classifier = results['fclf']
# Make prediction on training set, to understand data distribution
prediction_src = classifier.predict(ds_src)
#prediction_src = results['predictions_ds']
true_predictions = np.array(prediction_src) == ds_src.targets
example_dist = dict()
# Extract feature selected from each dataset
if isinstance(classifier, FeatureSelectionClassifier):
f_selection = results['fclf'].mapper
ds_tar = f_selection(ds_tar)
ds_src = f_selection(ds_src)
# Get no. of features
df = ds_src.samples.shape[1]
print 'dof ' + str(df)
#Set a t distribution
t_stu = scipy.stats.t(df)
#Set the p-value and the threshold value to validate predictions
m_value = t_stu.isf(p_value * 0.5)
'''
Get class distribution information: mean and covariance
'''
for label in np.unique(ds_src.targets):
# Get examples correctly classified
mask = ds_src.targets == label
example_dist[label] = dict()
true_ex = ds_src.samples[mask * true_predictions]
# Get Mean and Covariance to draw the distribution
mean_ = np.mean(true_ex, axis=0)
example_dist[label]['mean'] = mean_
std_ = np.std(true_ex, axis=0)
example_dist[label]['std'] = std_.mean()
example_dist[label]['threshold'] = euclidean(
mean_, mean_ + m_value * std_.mean())
# Get target predictions (unlabelled)
prediction_target = results['predictions']
# Test of data prediction
euclidean_values = []
normalized_values = []
true_predictions = []
for l, ex in zip(prediction_target, ds_tar.samples):
# Keep mahalanobis distance between examples and class distribution
dist = euclidean(example_dist[l]['mean'], ex)
euclidean_values.append(dist)
std_ = example_dist[l]['std']
true_predictions.append(dist < example_dist[l]['threshold'])
normalized_values.append(dist / np.sqrt(np.sum(np.power(std_, 2))))
distances = dict()
'''
threshold = euclidean(example_dist['trained']['mean'],
example_dist['untrained']['mean'])
'''
for c in np.unique(prediction_target):
mean_ = example_dist[c]['mean']
std_ = example_dist[c]['std']
distances[c] = []
for ex in ds_tar.samples:
distances[c].append(euclidean(mean_, ex))
distances[c] = np.array(distances[c])
threshold = euclidean(mean_, mean_ + m_value * std_.mean())
values = np.array(distances[c])
print values.shape
#true_predictions += (values < threshold)
'''
Squared Mahalanobis distance is similar to a chi square distribution with
degrees of freedom equal to the number of features.
'''
#mahalanobis_values = np.array(mahalanobis_values) ** 2
print 'threshold ' + str(m_value)
#print p_value
#Mask true predictions
p_values = 1 - 2 * t_stu.cdf(np.array(normalized_values))
#print np.count_nonzero(p_values)
'''
Get some data
'''
full_data = np.array(
zip(ds_tar.targets, prediction_target, euclidean_values, p_values))
#print np.count_nonzero(np.float_(full_data.T[3]) == p_values)
#true_data = full_data[true_predictions]
return full_data, np.bool_(
true_predictions), threshold, p_values, distances
def setData(self, index, value, role=Qt.DisplayRole):
"""Set the value to the index position depending on Qt::ItemDataRole and data type of the column
Args:
index (QtCore.QModelIndex): Index to define column and row.
value (object): new value.
role (Qt::ItemDataRole): Use this role to specify what you want to do.
Raises:
TypeError: If the value could not be converted to a known datatype.
Returns:
True if value is changed. Calls layoutChanged after update.
False if value is not different from original value.
"""
if not index.isValid() or not self.editable:
return False
if value != index.data(role):
self.layoutAboutToBeChanged.emit()
row = self._dataFrame.index[index.row()]
col = self._dataFrame.columns[index.column()]
#print 'before change: ', index.data().toUTC(), self._dataFrame.iloc[row][col]
columnDtype = self._dataFrame[col].dtype
if columnDtype == object:
pass
elif columnDtype in self._intDtypes:
dtypeInfo = numpy.iinfo(columnDtype)
if value < dtypeInfo.min:
value = dtypeInfo.min
elif value > dtypeInfo.max:
value = dtypeInfo.max
elif columnDtype in self._floatDtypes:
value = numpy.float64(value).astype(columnDtype)
elif columnDtype in self._boolDtypes:
value = numpy.bool_(value)
elif columnDtype in self._dateDtypes:
# convert the given value to a compatible datetime object.
# if the conversation could not be done, keep the original
# value.
if isinstance(value, QtCore.QDateTime):
value = value.toString(self.timestampFormat)
try:
value = pandas.Timestamp(value)
except Exception:
raise Exception(
"Can't convert '{0}' into a datetime".format(value))
# return False
else:
raise TypeError("try to set unhandled data type")
self._dataFrame.set_value(row, col, value)
#print 'after change: ', value, self._dataFrame.iloc[row][col]
self.layoutChanged.emit()
return True
else:
return False
scalar=(p.Scalar(), p.Scalar, -1),
array=(p.Array(shape=(3, 2)), p.Array, -1),
choice=(p.Choice([1, 2, 3]), p.Choice, 3),
choice_weight=(p.Choice([1, 2, 3]).weights, p.Choice, 3),
)
def test_parameter_as_choice_tag(param: p.Parameter, cls: tp.Type[p.Parameter],
arity: int) -> None:
tag = p.BaseChoice.ChoiceTag.as_tag(param)
assert tag.cls == cls
assert tag.arity == arity
@testing.parametrized(
true=(True, 0.0),
false=(False, 1.0),
np_true=(np.bool_(True), 0.0),
np_false=(np.bool_(False), 1.0),
pos=(0.7, 0.0),
neg=(-0.7, 0.7),
np_pos=(np.float(0.7), 0.0),
np_neg=(np.float(-0.7), 0.7),
)
def test_float_penalty(value: tp.Any, expected: float) -> None:
assert utils.float_penalty(value) == expected
def do_nothing(*args: tp.Any, **kwargs: tp.Any) -> int:
print("my args", args, flush=True)
print("my kwargs", kwargs, flush=True)
if "sleep" in kwargs:
print("Waiting", flush=True)
def process(self, df):
if not df.size:
return self.accumulator.identity()
self._configure(df)
dataset = df['dataset']
df['is_lo_w'] = is_lo_w(dataset)
df['is_lo_z'] = is_lo_z(dataset)
df['is_lo_znunu'] = is_lo_znunu(dataset)
df['is_lo_w_ewk'] = is_lo_w_ewk(dataset)
df['is_lo_z_ewk'] = is_lo_z_ewk(dataset)
df['is_lo_g'] = is_lo_g(dataset)
df['is_nlo_z'] = is_nlo_z(dataset)
df['is_nlo_w'] = is_nlo_w(dataset)
df['has_lhe_v_pt'] = df['is_lo_w'] | df['is_lo_z'] | df[
'is_nlo_z'] | df['is_nlo_w'] | df['is_lo_g'] | df[
'is_lo_w_ewk'] | df['is_lo_z_ewk']
df['is_data'] = is_data(dataset)
gen_v_pt = None
if df['is_lo_w'] or df['is_lo_z'] or df['is_nlo_z'] or df[
'is_nlo_w'] or df['is_lo_z_ewk'] or df['is_lo_w_ewk']:
gen = setup_gen_candidates(df)
dressed = setup_dressed_gen_candidates(df)
fill_gen_v_info(df, gen, dressed)
gen_v_pt = df['gen_v_pt_combined']
elif df['is_lo_g']:
gen = setup_gen_candidates(df)
all_gen_photons = gen[(gen.pdg == 22)]
prompt_mask = (all_gen_photons.status
== 1) & (all_gen_photons.flag & 1 == 1)
stat1_mask = (all_gen_photons.status == 1)
gen_photons = all_gen_photons[prompt_mask |
(~prompt_mask.any()) & stat1_mask]
gen_photon = gen_photons[gen_photons.pt.argmax()]
gen_v_pt = gen_photon.pt.max()
# Generator-level leading dijet mass
if df['has_lhe_v_pt']:
genjets = setup_lhe_cleaned_genjets(df)
digenjet = genjets[:, :2].distincts()
df['mjj_gen'] = digenjet.mass.max()
df['mjj_gen'] = np.where(df['mjj_gen'] > 0, df['mjj_gen'], 0)
# Candidates
# Already pre-filtered!
# All leptons are at least loose
# Check out setup_candidates for filtering details
met_pt, met_phi, ak4, bjets, _, muons, electrons, taus, photons = setup_candidates(
df, cfg)
# Remove jets in accordance with the noise recipe
if df['year'] == 2017:
ak4 = ak4[(ak4.ptraw > 50) | (ak4.abseta < 2.65) |
(ak4.abseta > 3.139)]
bjets = bjets[(bjets.ptraw > 50) | (bjets.abseta < 2.65) |
(bjets.abseta > 3.139)]
# Filtering ak4 jets according to pileup ID
ak4 = ak4[ak4.puid]
# Muons
df['is_tight_muon'] = muons.tightId \
& (muons.iso < cfg.MUON.CUTS.TIGHT.ISO) \
& (muons.pt>cfg.MUON.CUTS.TIGHT.PT) \
& (muons.abseta<cfg.MUON.CUTS.TIGHT.ETA)
dimuons = muons.distincts()
dimuon_charge = dimuons.i0['charge'] + dimuons.i1['charge']
df['MT_mu'] = ((muons.counts == 1) *
mt(muons.pt, muons.phi, met_pt, met_phi)).max()
# Electrons
df['is_tight_electron'] = electrons.tightId \
& (electrons.pt > cfg.ELECTRON.CUTS.TIGHT.PT) \
& (electrons.absetasc < cfg.ELECTRON.CUTS.TIGHT.ETA)
dielectrons = electrons.distincts()
dielectron_charge = dielectrons.i0['charge'] + dielectrons.i1['charge']
df['MT_el'] = ((electrons.counts == 1) *
mt(electrons.pt, electrons.phi, met_pt, met_phi)).max()
# ak4
leadak4_index = ak4.pt.argmax()
elejet_pairs = ak4[:, :1].cross(electrons)
df['dREleJet'] = np.hypot(
elejet_pairs.i0.eta - elejet_pairs.i1.eta,
dphi(elejet_pairs.i0.phi, elejet_pairs.i1.phi)).min()
muonjet_pairs = ak4[:, :1].cross(muons)
df['dRMuonJet'] = np.hypot(
muonjet_pairs.i0.eta - muonjet_pairs.i1.eta,
dphi(muonjet_pairs.i0.phi, muonjet_pairs.i1.phi)).min()
# Recoil
df['recoil_pt'], df['recoil_phi'] = recoil(met_pt, met_phi, electrons,
muons, photons)
df["dPFCaloSR"] = (met_pt - df["CaloMET_pt"]) / met_pt
df["dPFCaloCR"] = (met_pt - df["CaloMET_pt"]) / df["recoil_pt"]
df["dPFTkSR"] = (met_pt - df["TkMET_pt"]) / met_pt
df["minDPhiJetRecoil"] = min_dphi_jet_met(ak4,
df['recoil_phi'],
njet=4,
ptmin=30,
etamax=5.0)
df["minDPhiJetMet"] = min_dphi_jet_met(ak4,
met_phi,
njet=4,
ptmin=30,
etamax=5.0)
selection = processor.PackedSelection()
# Triggers
pass_all = np.ones(df.size) == 1
selection.add('inclusive', pass_all)
selection = trigger_selection(selection, df, cfg)
selection.add('mu_pt_trig_safe', muons.pt.max() > 30)
# Common selection
selection.add('veto_ele', electrons.counts == 0)
selection.add('veto_muo', muons.counts == 0)
selection.add('veto_photon', photons.counts == 0)
selection.add('veto_tau', taus.counts == 0)
selection.add('at_least_one_tau', taus.counts > 0)
selection.add('mindphijr',
df['minDPhiJetRecoil'] > cfg.SELECTION.SIGNAL.MINDPHIJR)
selection.add('mindphijm',
df['minDPhiJetMet'] > cfg.SELECTION.SIGNAL.MINDPHIJR)
# B jets are treated using veto weights
# So accept them in MC, but reject in data
if df['is_data']:
selection.add('veto_b', bjets.counts == 0)
else:
selection.add('veto_b', pass_all)
selection.add('dpfcalo_sr',
np.abs(df['dPFCaloSR']) < cfg.SELECTION.SIGNAL.DPFCALO)
selection.add('dpfcalo_cr',
np.abs(df['dPFCaloCR']) < cfg.SELECTION.SIGNAL.DPFCALO)
selection.add('recoil', df['recoil_pt'] > cfg.SELECTION.SIGNAL.RECOIL)
selection.add('met_sr', met_pt > cfg.SELECTION.SIGNAL.RECOIL)
# AK4 dijet
diak4 = ak4[:, :2].distincts()
leadak4_pt_eta = (diak4.i0.pt > cfg.SELECTION.SIGNAL.LEADAK4.PT) & (
np.abs(diak4.i0.eta) < cfg.SELECTION.SIGNAL.LEADAK4.ETA)
trailak4_pt_eta = (diak4.i1.pt > cfg.SELECTION.SIGNAL.TRAILAK4.PT) & (
np.abs(diak4.i1.eta) < cfg.SELECTION.SIGNAL.TRAILAK4.ETA)
hemisphere = (diak4.i0.eta * diak4.i1.eta < 0).any()
has_track0 = np.abs(diak4.i0.eta) <= 2.5
has_track1 = np.abs(diak4.i1.eta) <= 2.5
leadak4_id = diak4.i0.tightId & (has_track0 * (
(diak4.i0.chf > cfg.SELECTION.SIGNAL.LEADAK4.CHF) &
(diak4.i0.nhf < cfg.SELECTION.SIGNAL.LEADAK4.NHF)) + ~has_track0)
trailak4_id = has_track1 * (
(diak4.i1.chf > cfg.SELECTION.SIGNAL.TRAILAK4.CHF) &
(diak4.i1.nhf < cfg.SELECTION.SIGNAL.TRAILAK4.NHF)) + ~has_track1
df['mjj'] = diak4.mass.max()
df['dphijj'] = dphi(diak4.i0.phi.min(), diak4.i1.phi.max())
df['detajj'] = np.abs(diak4.i0.eta - diak4.i1.eta).max()
leading_jet_in_horn = ((diak4.i0.abseta < 3.2) &
(diak4.i0.abseta > 2.8)).any()
trailing_jet_in_horn = ((diak4.i1.abseta < 3.2) &
(diak4.i1.abseta > 2.8)).any()
selection.add('hornveto', (df['dPFTkSR'] < 0.8)
| ~(leading_jet_in_horn | trailing_jet_in_horn))
if df['year'] == 2018:
if df['is_data']:
metphihem_mask = ~((met_phi > -1.8) & (met_phi < -0.6) &
(df['run'] > 319077))
else:
metphihem_mask = pass_all
selection.add("metphihemextveto", metphihem_mask)
selection.add('no_el_in_hem',
electrons[electrons_in_hem(electrons)].counts == 0)
else:
selection.add("metphihemextveto", pass_all)
selection.add('no_el_in_hem', pass_all)
selection.add('two_jets', diak4.counts > 0)
selection.add('leadak4_pt_eta', leadak4_pt_eta.any())
selection.add('trailak4_pt_eta', trailak4_pt_eta.any())
selection.add('hemisphere', hemisphere)
selection.add('leadak4_id', leadak4_id.any())
selection.add('trailak4_id', trailak4_id.any())
selection.add('mjj',
df['mjj'] > cfg.SELECTION.SIGNAL.DIJET.SHAPE_BASED.MASS)
selection.add(
'dphijj',
df['dphijj'] < cfg.SELECTION.SIGNAL.DIJET.SHAPE_BASED.DPHI)
selection.add(
'detajj',
df['detajj'] > cfg.SELECTION.SIGNAL.DIJET.SHAPE_BASED.DETA)
# Cleaning cuts for signal region
# NEF cut: Only for endcap jets, require NEF < 0.7
ak40_in_endcap = (diak4.i0.abseta > 2.5) & (diak4.i0.abseta < 3.0)
ak41_in_endcap = (diak4.i1.abseta > 2.5) & (diak4.i1.abseta < 3.0)
max_neEmEF_ak40 = (~ak40_in_endcap) | (diak4.i0.nef < 0.7)
max_neEmEF_ak41 = (~ak41_in_endcap) | (diak4.i1.nef < 0.7)
max_neEmEF = (max_neEmEF_ak40 & max_neEmEF_ak41).any()
selection.add('max_neEmEF', max_neEmEF)
vec_b = calculate_vecB(ak4, met_pt, met_phi)
vec_dphi = calculate_vecDPhi(ak4, met_pt, met_phi, df['TkMET_phi'])
no_jet_in_trk = (diak4.i0.abseta > 2.5).any() & (diak4.i1.abseta >
2.5).any()
no_jet_in_hf = (diak4.i0.abseta < 3.0).any() & (diak4.i1.abseta <
3.0).any()
at_least_one_jet_in_hf = (diak4.i0.abseta >
3.0).any() | (diak4.i1.abseta > 3.0).any()
at_least_one_jet_in_trk = (diak4.i0.abseta <
2.5).any() | (diak4.i1.abseta < 2.5).any()
# Categorized cleaning cuts
eemitigation = ((no_jet_in_hf | at_least_one_jet_in_trk) &
(vec_dphi < 1.0)) | (
(no_jet_in_trk & at_least_one_jet_in_hf) &
(vec_b < 0.2))
selection.add('eemitigation', eemitigation)
# HF-HF veto in SR
both_jets_in_hf = (diak4.i0.abseta > 3.0) & (diak4.i1.abseta > 3.0)
selection.add('veto_hfhf', ~both_jets_in_hf.any())
# Divide into three categories for trigger study
if cfg.RUN.TRIGGER_STUDY:
two_central_jets = (np.abs(diak4.i0.eta) <= 2.4) & (np.abs(
diak4.i1.eta) <= 2.4)
two_forward_jets = (np.abs(diak4.i0.eta) > 2.4) & (np.abs(
diak4.i1.eta) > 2.4)
one_jet_forward_one_jet_central = (~two_central_jets) & (
~two_forward_jets)
selection.add('two_central_jets', two_central_jets.any())
selection.add('two_forward_jets', two_forward_jets.any())
selection.add('one_jet_forward_one_jet_central',
one_jet_forward_one_jet_central.any())
# Dimuon CR
leadmuon_index = muons.pt.argmax()
selection.add('at_least_one_tight_mu', df['is_tight_muon'].any())
selection.add('dimuon_mass', ((dimuons.mass > cfg.SELECTION.CONTROL.DOUBLEMU.MASS.MIN) \
& (dimuons.mass < cfg.SELECTION.CONTROL.DOUBLEMU.MASS.MAX)).any())
selection.add('dimuon_charge', (dimuon_charge == 0).any())
selection.add('two_muons', muons.counts == 2)
# Single muon CR
selection.add('one_muon', muons.counts == 1)
selection.add('mt_mu', df['MT_mu'] < cfg.SELECTION.CONTROL.SINGLEMU.MT)
# Diele CR
leadelectron_index = electrons.pt.argmax()
selection.add('one_electron', electrons.counts == 1)
selection.add('two_electrons', electrons.counts == 2)
selection.add('at_least_one_tight_el', df['is_tight_electron'].any())
selection.add('dielectron_mass', ((dielectrons.mass > cfg.SELECTION.CONTROL.DOUBLEEL.MASS.MIN) \
& (dielectrons.mass < cfg.SELECTION.CONTROL.DOUBLEEL.MASS.MAX)).any())
selection.add('dielectron_charge', (dielectron_charge == 0).any())
# Single Ele CR
selection.add('met_el', met_pt > cfg.SELECTION.CONTROL.SINGLEEL.MET)
selection.add('mt_el', df['MT_el'] < cfg.SELECTION.CONTROL.SINGLEEL.MT)
# Photon CR
leadphoton_index = photons.pt.argmax()
df['is_tight_photon'] = photons.mediumId & photons.barrel
selection.add('one_photon', photons.counts == 1)
selection.add('at_least_one_tight_photon', df['is_tight_photon'].any())
selection.add('photon_pt', photons.pt.max() > cfg.PHOTON.CUTS.TIGHT.PT)
selection.add('photon_pt_trig',
photons.pt.max() > cfg.PHOTON.CUTS.TIGHT.PTTRIG)
# Fill histograms
output = self.accumulator.identity()
# Gen
if df['has_lhe_v_pt']:
output['genvpt_check'].fill(vpt=gen_v_pt,
type="Nano",
dataset=dataset)
if 'LHE_Njets' in df:
output['lhe_njets'].fill(dataset=dataset,
multiplicity=df['LHE_Njets'])
if 'LHE_HT' in df:
output['lhe_ht'].fill(dataset=dataset, ht=df['LHE_HT'])
if 'LHE_HTIncoming' in df:
output['lhe_htinc'].fill(dataset=dataset, ht=df['LHE_HTIncoming'])
# Weights
evaluator = evaluator_from_config(cfg)
weights = processor.Weights(size=df.size, storeIndividual=True)
if not df['is_data']:
weights.add('gen', df['Generator_weight'])
try:
weights.add('prefire', df['PrefireWeight'])
except KeyError:
weights.add('prefire', np.ones(df.size))
weights = candidate_weights(weights, df, evaluator, muons,
electrons, photons, cfg)
# B jet veto weights
bsf_variations = btag_weights(bjets, cfg)
weights.add("bveto", (1 - bsf_variations["central"]).prod())
weights = pileup_weights(weights, df, evaluator, cfg)
weights = ak4_em_frac_weights(weights, diak4, evaluator)
if not (gen_v_pt is None):
weights = theory_weights_vbf(weights, df, evaluator, gen_v_pt,
df['mjj_gen'])
# Save per-event values for synchronization
if cfg.RUN.KINEMATICS.SAVE:
for event in cfg.RUN.KINEMATICS.EVENTS:
mask = df['event'] == event
if not mask.any():
continue
output['kinematics']['event'] += [event]
output['kinematics']['met'] += [met_pt[mask]]
output['kinematics']['met_phi'] += [met_phi[mask]]
output['kinematics']['recoil'] += [df['recoil_pt'][mask]]
output['kinematics']['recoil_phi'] += [df['recoil_phi'][mask]]
output['kinematics']['ak4pt0'] += [ak4[leadak4_index][mask].pt]
output['kinematics']['ak4eta0'] += [
ak4[leadak4_index][mask].eta
]
output['kinematics']['leadbtag'] += [ak4.pt.max() < 0][mask]
output['kinematics']['nLooseMu'] += [muons.counts[mask]]
output['kinematics']['nTightMu'] += [
muons[df['is_tight_muon']].counts[mask]
]
output['kinematics']['mupt0'] += [
muons[leadmuon_index][mask].pt
]
output['kinematics']['mueta0'] += [
muons[leadmuon_index][mask].eta
]
output['kinematics']['nLooseEl'] += [electrons.counts[mask]]
output['kinematics']['nTightEl'] += [
electrons[df['is_tight_electron']].counts[mask]
]
output['kinematics']['elpt0'] += [
electrons[leadelectron_index][mask].pt
]
output['kinematics']['eleta0'] += [
electrons[leadelectron_index][mask].eta
]
output['kinematics']['nLooseGam'] += [photons.counts[mask]]
output['kinematics']['nTightGam'] += [
photons[df['is_tight_photon']].counts[mask]
]
output['kinematics']['gpt0'] += [
photons[leadphoton_index][mask].pt
]
output['kinematics']['geta0'] += [
photons[leadphoton_index][mask].eta
]
# Sum of all weights to use for normalization
# TODO: Deal with systematic variations
output['nevents'][dataset] += df.size
if not df['is_data']:
output['sumw'][dataset] += df['genEventSumw']
output['sumw2'][dataset] += df['genEventSumw2']
output['sumw_pileup'][dataset] += weights._weights['pileup'].sum()
regions = vbfhinv_regions(cfg)
# Get veto weights (only for MC)
if not df['is_data']:
veto_weights = get_veto_weights(df, cfg, evaluator, electrons,
muons, taus)
for region, cuts in regions.items():
exclude = [None]
region_weights = copy.deepcopy(weights)
if not df['is_data']:
### Trigger weights
if re.match(r'cr_(\d+)e.*', region):
p_pass_data = 1 - (1 -
evaluator["trigger_electron_eff_data"]
(electrons.etasc, electrons.pt)).prod()
p_pass_mc = 1 - (1 - evaluator["trigger_electron_eff_mc"]
(electrons.etasc, electrons.pt)).prod()
trigger_weight = p_pass_data / p_pass_mc
trigger_weight[np.isnan(trigger_weight)] = 1
region_weights.add('trigger', trigger_weight)
elif re.match(r'cr_(\d+)m.*', region) or re.match(
'sr_.*', region):
met_trigger_sf(
region_weights,
diak4,
df,
apply_categorized=cfg.RUN.APPLY_CATEGORIZED_SF)
elif re.match(r'cr_g.*', region):
photon_trigger_sf(region_weights, photons, df)
# Veto weights
if re.match('.*no_veto.*', region):
exclude = [
"muon_id_iso_tight", "muon_id_tight", "muon_iso_tight",
"muon_id_loose", "muon_iso_loose", "ele_reco",
"ele_id_tight", "ele_id_loose", "tau_id"
]
region_weights.add(
"veto",
veto_weights.partial_weight(include=["nominal"]))
# HEM-veto weights for signal region MC
if re.match('^sr_vbf.*', region) and df['year'] == 2018:
# Events that lie in the HEM-veto region
events_to_weight_mask = (met_phi > -1.8) & (met_phi < -0.6)
# Weight is the "good lumi fraction" for 2018
weight = 21.1 / 59.7
hem_weight = np.where(events_to_weight_mask, weight, 1.0)
region_weights.add("hem_weight", hem_weight)
# This is the default weight for this region
rweight = region_weights.partial_weight(exclude=exclude)
# Blinding
if (self._blind and df['is_data'] and region.startswith('sr')):
continue
# Cutflow plot for signal and control regions
if any(x in region for x in ["sr", "cr", "tr"]):
output['cutflow_' + region][dataset]['all'] += df.size
for icut, cutname in enumerate(cuts):
output['cutflow_' +
region][dataset][cutname] += selection.all(
*cuts[:icut + 1]).sum()
mask = selection.all(*cuts)
if cfg.RUN.SAVE.TREE:
if region in ['cr_1e_vbf', 'cr_1m_vbf']:
output['tree_int64'][region][
"event"] += processor.column_accumulator(
df["event"][mask])
output['tree_float16'][region][
"gen_v_pt"] += processor.column_accumulator(
np.float16(gen_v_pt[mask]))
output['tree_float16'][region][
"gen_mjj"] += processor.column_accumulator(
np.float16(df['mjj_gen'][mask]))
output['tree_float16'][region][
"recoil_pt"] += processor.column_accumulator(
np.float16(df["recoil_pt"][mask]))
output['tree_float16'][region][
"recoil_phi"] += processor.column_accumulator(
np.float16(df["recoil_phi"][mask]))
output['tree_float16'][region][
"mjj"] += processor.column_accumulator(
np.float16(df["mjj"][mask]))
output['tree_float16'][region][
"leadak4_pt"] += processor.column_accumulator(
np.float16(diak4.i0.pt[mask]))
output['tree_float16'][region][
"leadak4_eta"] += processor.column_accumulator(
np.float16(diak4.i0.eta[mask]))
output['tree_float16'][region][
"leadak4_phi"] += processor.column_accumulator(
np.float16(diak4.i0.phi[mask]))
output['tree_float16'][region][
"trailak4_pt"] += processor.column_accumulator(
np.float16(diak4.i1.pt[mask]))
output['tree_float16'][region][
"trailak4_eta"] += processor.column_accumulator(
np.float16(diak4.i1.eta[mask]))
output['tree_float16'][region][
"trailak4_phi"] += processor.column_accumulator(
np.float16(diak4.i1.phi[mask]))
output['tree_float16'][region][
"minDPhiJetRecoil"] += processor.column_accumulator(
np.float16(df["minDPhiJetRecoil"][mask]))
if '_1e_' in region:
output['tree_float16'][region][
"leadlep_pt"] += processor.column_accumulator(
np.float16(electrons.pt.max()[mask]))
output['tree_float16'][region][
"leadlep_eta"] += processor.column_accumulator(
np.float16(electrons[
electrons.pt.argmax()].eta.max()[mask]))
output['tree_float16'][region][
"leadlep_phi"] += processor.column_accumulator(
np.float16(electrons[
electrons.pt.argmax()].phi.max()[mask]))
elif '_1m_' in region:
output['tree_float16'][region][
"leadlep_pt"] += processor.column_accumulator(
np.float16(muons.pt.max()[mask]))
output['tree_float16'][region][
"leadlep_eta"] += processor.column_accumulator(
np.float16(
muons[muons.pt.argmax()].eta.max()[mask]))
output['tree_float16'][region][
"leadlep_phi"] += processor.column_accumulator(
np.float16(
muons[muons.pt.argmax()].phi.max()[mask]))
for name, w in region_weights._weights.items():
output['tree_float16'][region][
f"weight_{name}"] += processor.column_accumulator(
np.float16(w[mask]))
output['tree_float16'][region][
f"weight_total"] += processor.column_accumulator(
np.float16(rweight[mask]))
if region == 'inclusive':
output['tree_int64'][region][
"event"] += processor.column_accumulator(
df["event"][mask])
for name in selection.names:
output['tree_bool'][region][
name] += processor.column_accumulator(
np.bool_(selection.all(*[name])[mask]))
# Save the event numbers of events passing this selection
# Save the event numbers of events passing this selection
if cfg.RUN.SAVE.PASSING:
output['selected_events'][region] += list(df['event'][mask])
# Multiplicities
def fill_mult(name, candidates):
output[name].fill(dataset=dataset,
region=region,
multiplicity=candidates[mask].counts,
weight=rweight[mask])
fill_mult('ak4_mult', ak4[ak4.pt > 30])
fill_mult('bjet_mult', bjets)
fill_mult('loose_ele_mult', electrons)
fill_mult('tight_ele_mult', electrons[df['is_tight_electron']])
fill_mult('loose_muo_mult', muons)
fill_mult('tight_muo_mult', muons[df['is_tight_muon']])
fill_mult('tau_mult', taus)
fill_mult('photon_mult', photons)
def ezfill(name, **kwargs):
"""Helper function to make filling easier."""
output[name].fill(dataset=dataset, region=region, **kwargs)
# Monitor weights
for wname, wvalue in region_weights._weights.items():
ezfill("weights", weight_type=wname, weight_value=wvalue[mask])
ezfill("weights_wide",
weight_type=wname,
weight_value=wvalue[mask])
# All ak4
# This is a workaround to create a weight array of the right dimension
w_alljets = weight_shape(ak4[mask].eta, rweight[mask])
w_alljets_nopref = weight_shape(
ak4[mask].eta,
region_weights.partial_weight(exclude=exclude +
['prefire'])[mask])
ezfill('ak4_eta', jeteta=ak4[mask].eta.flatten(), weight=w_alljets)
ezfill('ak4_phi', jetphi=ak4[mask].phi.flatten(), weight=w_alljets)
ezfill('ak4_pt', jetpt=ak4[mask].pt.flatten(), weight=w_alljets)
ezfill('ak4_eta_nopref',
jeteta=ak4[mask].eta.flatten(),
weight=w_alljets_nopref)
ezfill('ak4_phi_nopref',
jetphi=ak4[mask].phi.flatten(),
weight=w_alljets_nopref)
ezfill('ak4_pt_nopref',
jetpt=ak4[mask].pt.flatten(),
weight=w_alljets_nopref)
# Leading ak4
w_diak4 = weight_shape(diak4.pt[mask], rweight[mask])
ezfill('ak4_eta0',
jeteta=diak4.i0.eta[mask].flatten(),
weight=w_diak4)
ezfill('ak4_phi0',
jetphi=diak4.i0.phi[mask].flatten(),
weight=w_diak4)
ezfill('ak4_pt0',
jetpt=diak4.i0.pt[mask].flatten(),
weight=w_diak4)
ezfill('ak4_ptraw0',
jetpt=diak4.i0.ptraw[mask].flatten(),
weight=w_diak4)
ezfill('ak4_chf0',
frac=diak4.i0.chf[mask].flatten(),
weight=w_diak4)
ezfill('ak4_nhf0',
frac=diak4.i0.nhf[mask].flatten(),
weight=w_diak4)
ezfill('ak4_nconst0',
nconst=diak4.i0.nconst[mask].flatten(),
weight=w_diak4)
# Trailing ak4
ezfill('ak4_eta1',
jeteta=diak4.i1.eta[mask].flatten(),
weight=w_diak4)
ezfill('ak4_phi1',
jetphi=diak4.i1.phi[mask].flatten(),
weight=w_diak4)
ezfill('ak4_pt1',
jetpt=diak4.i1.pt[mask].flatten(),
weight=w_diak4)
ezfill('ak4_ptraw1',
jetpt=diak4.i1.ptraw[mask].flatten(),
weight=w_diak4)
ezfill('ak4_chf1',
frac=diak4.i1.chf[mask].flatten(),
weight=w_diak4)
ezfill('ak4_nhf1',
frac=diak4.i1.nhf[mask].flatten(),
weight=w_diak4)
ezfill('ak4_nconst1',
nconst=diak4.i1.nconst[mask].flatten(),
weight=w_diak4)
# B tag discriminator
btag = getattr(ak4, cfg.BTAG.ALGO)
w_btag = weight_shape(btag[mask], rweight[mask])
ezfill('ak4_btag', btag=btag[mask].flatten(), weight=w_btag)
# MET
ezfill('dpfcalo_cr',
dpfcalo=df["dPFCaloCR"][mask],
weight=rweight[mask])
ezfill('dpfcalo_sr',
dpfcalo=df["dPFCaloSR"][mask],
weight=rweight[mask])
ezfill('met', met=met_pt[mask], weight=rweight[mask])
ezfill('met_phi', phi=met_phi[mask], weight=rweight[mask])
ezfill('recoil',
recoil=df["recoil_pt"][mask],
weight=rweight[mask])
ezfill('recoil_phi',
phi=df["recoil_phi"][mask],
weight=rweight[mask])
ezfill('dphijm',
dphi=df["minDPhiJetMet"][mask],
weight=rweight[mask])
ezfill('dphijr',
dphi=df["minDPhiJetRecoil"][mask],
weight=rweight[mask])
ezfill('dphijj', dphi=df["dphijj"][mask], weight=rweight[mask])
ezfill('detajj', deta=df["detajj"][mask], weight=rweight[mask])
ezfill('mjj', mjj=df["mjj"][mask], weight=rweight[mask])
# b-tag weight up and down variations
if cfg.RUN.BTAG_STUDY:
if not df['is_data']:
rw = region_weights.partial_weight(exclude=exclude +
['bveto'])
ezfill('mjj_bveto_up',
mjj=df['mjj'][mask],
weight=(rw *
(1 - bsf_variations['up']).prod())[mask])
ezfill('mjj_bveto_down',
mjj=df['mjj'][mask],
weight=(rw *
(1 - bsf_variations['down']).prod())[mask])
if gen_v_pt is not None:
ezfill('gen_vpt',
vpt=gen_v_pt[mask],
weight=df['Generator_weight'][mask])
ezfill('gen_mjj',
mjj=df['mjj_gen'][mask],
weight=df['Generator_weight'][mask])
# Photon CR data-driven QCD estimate
if df['is_data'] and re.match("cr_g.*", region) and re.match(
"(SinglePhoton|EGamma).*", dataset):
w_imp = photon_impurity_weights(
photons[leadphoton_index].pt.max()[mask], df["year"])
output['mjj'].fill(dataset=data_driven_qcd_dataset(dataset),
region=region,
mjj=df["mjj"][mask],
weight=rweight[mask] * w_imp)
output['recoil'].fill(dataset=data_driven_qcd_dataset(dataset),
region=region,
recoil=df["recoil_pt"][mask],
weight=rweight[mask] * w_imp)
# Uncertainty variations
if df['is_lo_z'] or df['is_nlo_z'] or df['is_lo_z_ewk']:
theory_uncs = [x for x in cfg.SF.keys() if x.startswith('unc')]
for unc in theory_uncs:
reweight = evaluator[unc](gen_v_pt)
w = (region_weights.weight() * reweight)[mask]
ezfill('mjj_unc',
mjj=df['mjj'][mask],
uncertainty=unc,
weight=w)
# Two dimensional
ezfill('recoil_mjj',
recoil=df["recoil_pt"][mask],
mjj=df["mjj"][mask],
weight=rweight[mask])
# Muons
if '_1m_' in region or '_2m_' in region or 'no_veto' in region:
w_allmu = weight_shape(muons.pt[mask], rweight[mask])
ezfill('muon_pt', pt=muons.pt[mask].flatten(), weight=w_allmu)
ezfill('muon_pt_abseta',
pt=muons.pt[mask].flatten(),
abseta=muons.eta[mask].flatten(),
weight=w_allmu)
ezfill('muon_mt', mt=df['MT_mu'][mask], weight=rweight[mask])
ezfill('muon_eta',
eta=muons.eta[mask].flatten(),
weight=w_allmu)
ezfill('muon_phi',
phi=muons.phi[mask].flatten(),
weight=w_allmu)
# Dimuon
if '_2m_' in region:
w_dimu = weight_shape(dimuons.pt[mask], rweight[mask])
ezfill('muon_pt0',
pt=dimuons.i0.pt[mask].flatten(),
weight=w_dimu)
ezfill('muon_pt1',
pt=dimuons.i1.pt[mask].flatten(),
weight=w_dimu)
ezfill('muon_eta0',
eta=dimuons.i0.eta[mask].flatten(),
weight=w_dimu)
ezfill('muon_eta1',
eta=dimuons.i1.eta[mask].flatten(),
weight=w_dimu)
ezfill('muon_phi0',
phi=dimuons.i0.phi[mask].flatten(),
weight=w_dimu)
ezfill('muon_phi1',
phi=dimuons.i1.phi[mask].flatten(),
weight=w_dimu)
ezfill('dimuon_pt',
pt=dimuons.pt[mask].flatten(),
weight=w_dimu)
ezfill('dimuon_eta',
eta=dimuons.eta[mask].flatten(),
weight=w_dimu)
ezfill('dimuon_mass',
dilepton_mass=dimuons.mass[mask].flatten(),
weight=w_dimu)
# Electrons
if '_1e_' in region or '_2e_' in region or 'no_veto' in region:
w_allel = weight_shape(electrons.pt[mask], rweight[mask])
ezfill('electron_pt',
pt=electrons.pt[mask].flatten(),
weight=w_allel)
ezfill('electron_pt_eta',
pt=electrons.pt[mask].flatten(),
eta=electrons.eta[mask].flatten(),
weight=w_allel)
ezfill('electron_mt',
mt=df['MT_el'][mask],
weight=rweight[mask])
ezfill('electron_eta',
eta=electrons.eta[mask].flatten(),
weight=w_allel)
ezfill('electron_phi',
phi=electrons.phi[mask].flatten(),
weight=w_allel)
# Dielectron
if '_2e_' in region:
w_diel = weight_shape(dielectrons.pt[mask], rweight[mask])
ezfill('electron_pt0',
pt=dielectrons.i0.pt[mask].flatten(),
weight=w_diel)
ezfill('electron_pt1',
pt=dielectrons.i1.pt[mask].flatten(),
weight=w_diel)
ezfill('electron_eta0',
eta=dielectrons.i0.eta[mask].flatten(),
weight=w_diel)
ezfill('electron_eta1',
eta=dielectrons.i1.eta[mask].flatten(),
weight=w_diel)
ezfill('electron_phi0',
phi=dielectrons.i0.phi[mask].flatten(),
weight=w_diel)
ezfill('electron_phi1',
phi=dielectrons.i1.phi[mask].flatten(),
weight=w_diel)
ezfill('dielectron_pt',
pt=dielectrons.pt[mask].flatten(),
weight=w_diel)
ezfill('dielectron_eta',
eta=dielectrons.eta[mask].flatten(),
weight=w_diel)
ezfill('dielectron_mass',
dilepton_mass=dielectrons.mass[mask].flatten(),
weight=w_diel)
# Photon
if '_g_' in region:
w_leading_photon = weight_shape(
photons[leadphoton_index].pt[mask], rweight[mask])
ezfill('photon_pt0',
pt=photons[leadphoton_index].pt[mask].flatten(),
weight=w_leading_photon)
ezfill('photon_eta0',
eta=photons[leadphoton_index].eta[mask].flatten(),
weight=w_leading_photon)
ezfill('photon_phi0',
phi=photons[leadphoton_index].phi[mask].flatten(),
weight=w_leading_photon)
ezfill('photon_pt0_recoil',
pt=photons[leadphoton_index].pt[mask].flatten(),
recoil=df['recoil_pt'][mask
& (leadphoton_index.counts > 0)],
weight=w_leading_photon)
ezfill('photon_eta_phi',
eta=photons[leadphoton_index].eta[mask].flatten(),
phi=photons[leadphoton_index].phi[mask].flatten(),
weight=w_leading_photon)
# w_drphoton_jet = weight_shape(df['dRPhotonJet'][mask], rweight[mask])
# Tau
if 'no_veto' in region:
w_all_taus = weight_shape(taus.pt[mask], rweight[mask])
ezfill("tau_pt", pt=taus.pt[mask].flatten(), weight=w_all_taus)
# PV
ezfill('npv', nvtx=df['PV_npvs'][mask], weight=rweight[mask])
ezfill('npvgood',
nvtx=df['PV_npvsGood'][mask],
weight=rweight[mask])
ezfill('npv_nopu',
nvtx=df['PV_npvs'][mask],
weight=region_weights.partial_weight(exclude=exclude +
['pileup'])[mask])
ezfill('npvgood_nopu',
nvtx=df['PV_npvsGood'][mask],
weight=region_weights.partial_weight(exclude=exclude +
['pileup'])[mask])
ezfill('rho_all',
rho=df['fixedGridRhoFastjetAll'][mask],
weight=region_weights.partial_weight(exclude=exclude)[mask])
ezfill('rho_central',
rho=df['fixedGridRhoFastjetCentral'][mask],
weight=region_weights.partial_weight(exclude=exclude)[mask])
ezfill('rho_all_nopu',
rho=df['fixedGridRhoFastjetAll'][mask],
weight=region_weights.partial_weight(exclude=exclude +
['pileup'])[mask])
ezfill('rho_central_nopu',
rho=df['fixedGridRhoFastjetCentral'][mask],
weight=region_weights.partial_weight(exclude=exclude +
['pileup'])[mask])
return output
@hash_object_dispatch.register((pd.DataFrame, pd.Series, pd.Index))
def hash_object_pandas(obj,
index=True,
encoding="utf8",
hash_key=None,
categorize=True):
return pd.util.hash_pandas_object(obj,
index=index,
encoding=encoding,
hash_key=hash_key,
categorize=categorize)
_simple_fake_mapping = {
"b": np.bool_(True),
"V": np.void(b" "),
"M": np.datetime64("1970-01-01"),
"m": np.timedelta64(1),
"S": np.str_("foo"),
"a": np.str_("foo"),
"U": np.unicode_("foo"),
"O": "foo",
}
def _scalar_from_dtype(dtype):
if dtype.kind in ("i", "f", "u"):
return dtype.type(1)
elif dtype.kind == "c":
return dtype.type(complex(1, 0))
# Program is part of MintPy # # Copyright(c) 2016-2019, Zhang Yunjun # # Author: Zhang Yunjun # ############################################################ import os import re import argparse import datetime as dt import h5py import numpy as np from mintpy.objects import timeseries, geometry, sensor from mintpy.utils import readfile from mintpy import info BOOL_ZERO = np.bool_(0) INT_ZERO = np.int16(0) FLOAT_ZERO = np.float32(0.0) CPX_ZERO = np.complex64(0.0) compression = 'lzf' ################################################################ TEMPALTE = """ mintpy.save.hdfEos5 = auto #[yes / no], auto for no, save timeseries to HDF-EOS5 format mintpy.save.hdfEos5.update = auto #[yes / no], auto for no, put XXXXXXXX as endDate in output filename mintpy.save.hdfEos5.subset = auto #[yes / no], auto for no, put subset range info in output filename """ EXAMPLE = """example: save_hdfeos5.py geo_timeseries_ECMWF_ramp_demErr.h5 -c geo_temporalCoherence.h5 -m geo_maskTempCoh.h5 -g geo_geometryRadar.h5 """