def test_ledger(self):
self.maxDiff = None
sio = StringIO()
with redirectStdoutCtx(sio):
FA._V1()
FA._V2()
FA._OFF()
FA._ON()
FA._TOFF()
FA._TON()
FA._LON()
FA._ROFF()
FA._RON()
FA._RDUMP()
FA._LOFF()
FA._LDUMP()
FA._LCLEAR()
FA._GCNOW()
FA._GCSET(100)
FA.Verbose = 3
FA._V1()
a = FA([1, 2, 3])
b = a.copy()
a += a
del a
a = np.arange(10000)
a = FA(a)
a = a + a
a = a + a
b = a._np
del b
del a
FA._V0()
def test_forced_dtype_binary(self):
'''
Test to see that datatype is being correctly forced for result of binary ufuncs.
You cannot force the type to be an unsigned integer.
You cannot force the type for a division operation.
'''
nums = [1, 2, 3, 4, 5]
types = int_types[1:] + float_types
types = [i for i in types
if i.__name__[0] != 'u'] # remove unsigned integers
binary_funcs = [
np.add,
np.subtract,
np.multiply,
] # , np.divide, np.floor_divide ]
for dt1 in types:
arr1 = FA(nums, dtype=dt1)
for dt2 in types:
arr2 = FA(nums, dtype=dt2)
for dt_forced in types:
for func in binary_funcs:
result = func(arr1, arr2, dtype=dt_forced)
self.assertEqual(
result.dtype,
dt_forced,
msg=
f"Unable to force dtype {dt_forced} for binary func {func} between {dt1} and {dt2}",
)
def test_rank(self):
a = FA([nan, 2, 2, 3])
b = a.nanrankdata()
assert_array_equal(b, [nan, 1.5, 1.5, 3.0])
a = FA([0, 2, 2, 3])
b = a.rankdata()
assert_array_equal(b, [1, 2.5, 2.5, 4])
def test_cut_errors(self):
with self.assertRaises(ValueError):
c = cut(
FA([2, 4, 6, 8, 10]),
FA([0, 2, 4, 6, 8, 10]),
labels=['a', 'b', 'c', 'd', 'e', 'f'],
)
def test_partition(self):
a = FA([1, 0, 3, 4, 2])
b = a.partition2(kth=2)
assert_array_equal(b, [1, 0, 2, 4, 3])
a = FA([10, 0, 30, 40, 20])
b = a.argpartition2(kth=2)
assert_array_equal(b, [0, 1, 4, 3, 2])
def test_qcut(self):
c = qcut(arange(10), 3)
self.assertTrue(sum(c._np - FA([2, 2, 2, 2, 3, 3, 3, 4, 4, 4])) == 0)
c = qcut(arange(11), 3)
self.assertTrue(
sum(c._np - FA([2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4])) == 0)
c = qcut(range(5), 3, labels=["good", "medium", "bad"])
self.assertTrue(sum(c._np - FA([2, 2, 3, 4, 4])) == 0)
c = qcut(arange(100.0), [0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95])
self.assertTrue(c._np[0] == 1)
self.assertTrue(c._np[5] == 2)
self.assertTrue(c._np[94] == 7)
self.assertTrue(c._np[95] == 1)
c = qcut(arange(100.0), [0.00, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95])
self.assertTrue(c._np[0] == 2)
self.assertTrue(c._np[5] == 2)
self.assertTrue(c._np[94] == 7)
self.assertTrue(c._np[95] == 1)
c = qcut(range(5), 4)
self.assertIsInstance(c, Categorical)
self.assertTrue(sum(c._np - FA([2, 2, 3, 4, 5])) == 0)
self.assertTrue((c.category_array == [
b'Clipped', b'0.0->1.0', b'1.0->2.0', b'2.0->3.0', b'3.0->4.0'
]).all())
c = qcut(range(5), 4, labels=True)
self.assertIsInstance(c, Categorical)
self.assertTrue(sum(c._np - FA([2, 2, 3, 4, 5])) == 0)
self.assertTrue((c.category_array == [
b'Clipped', b'0.0->1.0', b'1.0->2.0', b'2.0->3.0', b'3.0->4.0'
]).all())
c = qcut(range(5), 4, labels=None)
self.assertIsInstance(c, Categorical)
self.assertTrue(sum(c._np - FA([2, 2, 3, 4, 5])) == 0)
self.assertTrue((c.category_array == [
b'Clipped', b'0.0->1.0', b'1.0->2.0', b'2.0->3.0', b'3.0->4.0'
]).all())
c = qcut(range(5), 3, labels=["good", "medium", "bad"])
self.assertIsInstance(c, Categorical)
self.assertTrue(sum(c._np - FA([2, 2, 3, 4, 4])) == 0)
self.assertTrue(
np.array([(h == t) for (h, t) in zip(
c.expand_array.astype('U'),
['good', 'good', 'medium', 'bad', 'bad'],
)]).all())
self.assertTrue(
(c.category_array == [[b'Clipped', b'good', b'medium',
b'bad']]).all())
c = qcut(range(5), 4, labels=False)
self.assertIsInstance(c, FastArray)
self.assertTrue(sum(c._np - FA([2, 2, 3, 4, 5])) == 0)
def test_dtype_int_reduce(self):
'''
If the result of a reduce function is an integer, it will be cast to int64.
If the input is uint64, the output will also be uint64
'''
func_names = ['nansum']
unsigned_arr = FA(num_list, dtype=np.int32)
signed_arr = FA(num_list, dtype=np.uint64)
for name in func_names:
us_func = unsigned_arr.__getattribute__(name)
s_func = signed_arr.__getattribute__(name)
us_dt = us_func().dtype
s_dt = s_func().dtype
self.assertEqual(us_dt, np.int64)
self.assertEqual(s_dt, np.uint64)
def test_np_vs_member(self):
import builtins
'''
Check to make sure the result is the same no matter how the ufunc is accessed.
'''
func_names = ['min', 'max', 'sum']
arr = FA([1, 2, 3, 4, 5])
num_types = int_types[1:] + float_types
for dt in num_types:
arr = arr.astype(dt)
# print(dt)
for name in func_names:
member_func = None
np_func = None
builtin_func = None
results = []
member_func = arr.__getattribute__(name)
results.append(member_func())
if hasattr(np, name):
np_func = np.__getattribute__(name)
results.append(np_func(arr))
if hasattr(builtins, name):
builtin_func = builtins.__getattribute__(name)
results.append(builtin_func(arr))
self.assertEqual(
len(set(results)),
1,
msg=
f"Results did not match for datatype {dt} and function {name}. Fastarray: {member_func} Numpy: {np_func} Builtin: {builtin_func}",
)
def test_reduce_accuracy(self):
'''
FastArray often performs math operations at a higher level of precision than numpy. As a
result, the answers may be slightly different.
'''
reduce_funcs = [
np.sum,
np.mean,
np.std,
np.var,
np.nansum,
np.nanmean,
np.nanstd,
np.nanvar,
]
# use a large array to illustrate the differences
num_list = (np.random.rand(1000000) * 100) + 1
num_types = int_types + float_types
for dt in num_types:
fa_arr = FA(num_list, dtype=dt)
np_arr = np.array(num_list, dtype=dt)
for func in reduce_funcs:
fa_result = func(fa_arr)
np_result = func(np_arr)
# 4 places of accuracy seems to solve differences in float32/64
# TODO: Change to use self.assertAlmostEqual() instead of isclose?
is_accurate = isclose(fa_result, np_result, rel_tol=1e-06)
self.assertTrue(
is_accurate,
msg=
f"Reduce function {func} produced fa: {fa_result} np: {np_result} for datatype {dt} with data: {fa_arr}",
)
def test_ddof_default(self):
'''
FastArray differs from numpy when calculating var, nanvar, std, and nanstd. We set ddof's default to 1 instead of 0
'''
arr = FA([1, 2])
func_names = ['std', 'var', 'nanstd', 'nanvar']
for name in func_names:
# make sure different than numpy
func = arr.__getattribute__(name)
fa_default = func()
np_default = func(ddof=0)
self.assertNotAlmostEqual(
fa_default,
np_default,
msg=f"Failed to set ddof to 1 for reduce function {name}",
)
# make sure ddof is 1 when sent to numpy reduce
sent_to_numpy = func(keepdims=0)
self.assertAlmostEqual(
fa_default,
sent_to_numpy,
msg=
f"Failed to set ddof to 1 before passing to numpy for function {name}",
)
def test_shift(self):
'''
Check to make sure FastArray's shift mimics pandas shift - not numpy roll.
'''
arr0 = FA([1, 2, 3, 4, 5])
all_ints = int_types # [1:] # bool is not included
shift_dict = {
tuple(float_types): np.nan,
tuple([np.str_]): '',
tuple([np.bytes_]): b'',
}
for tp in all_ints:
if tp is bool:
tp = np.bool_
shift_dict[(tp, )] = INVALID_DICT[tp(1).dtype.num]
for dtypes, invalid in shift_dict.items():
for dt in dtypes:
arr = arr0.astype(dt)
pos_shift = arr.shift(1)
neg_shift = arr.shift(-1)
if invalid != invalid:
self.assertNotEqual(
pos_shift[0],
pos_shift[0],
msg=
f"Positive shift on datatype {dt} did not fill with {invalid}.",
)
self.assertNotEqual(
neg_shift[-1],
neg_shift[-1],
msg=
f"Negative shift on datatype {dt} did not fill with {invalid}.",
)
else:
self.assertEqual(
pos_shift[0],
invalid,
msg=
f"Positive shift on datatype {dt} did not fill with {invalid}.",
)
self.assertEqual(
neg_shift[-1],
invalid,
msg=
f"Negative shift on datatype {dt} did not fill with {invalid}.",
)
self.assertEqual(
pos_shift[1],
arr[0],
msg=
f"Positive shift on datatype {dt} did not shift existing values to the correct place.",
)
self.assertEqual(
neg_shift[0],
arr[1],
msg=
f"Negative shift on datatype {dt} did not shift existing values to the correct place.",
)
def check_exception(self):
'''
Example of checking that we raise the same error as numpy
'''
try:
n = np.array()
except Exception as e:
with self.assertRaises(e.__class__):
f = FA()
else:
f = FA()
self.assertEqual(
f,
n,
msg=
"Empty constructor produced a different result for numpy and fastarray",
)
def test_count(self):
"""
Tests that Categorical.count(transform=True) works correctly.
"""
x = Cat(np.arange(10) % 3)
y = x.count(transform=True)[0]
assert_array_equal(y, FA([4, 3, 3, 4, 3, 3, 4, 3, 3, 4]))
def test_qcut_drop_dups(self):
c = qcut([1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 11, 12, 13],
4,
duplicates='drop')
self.assertIsInstance(c, Categorical)
self.assertTrue(
sum(c._np - FA([2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3])) == 0)
self.assertTrue(
(c.category_array == [b'Clipped', b'1.0->2.0',
b'2.0->13.0']).all())
def test_dtype_reduce(self):
'''
If a dtype is passed to a reduce function, make sure the result is the correct dtype
'''
dt = np.int8
arr = FA([1, 2])
func_names = ['std', 'var', 'nanstd', 'nanvar']
for name in func_names:
func = arr.__getattribute__(name)
result = func(dtype=dt)
self.assertEqual(
dt,
result.dtype,
msg=
f"Dtypes did not match for func {name}. {dt} was the keyword, the output was {result.dtype}",
)
def test_display_properties(self):
'''
FastArrays of default types have default item formatting for display (see Utils.rt_display_properties)
This checks to see that the correct item format is being returned from a FastArray
'''
f = FA(num_list, dtype=np.int32)
item_format, convert_func = f.display_query_properties()
self.assertEqual(
item_format.length,
DisplayLength.Short,
msg=f"Incorrect length for item format.",
)
self.assertEqual(item_format.justification, DisplayJustification.Right)
# self.assertEqual(item_format.invalid, None)
self.assertEqual(item_format.can_have_spaces, False)
self.assertEqual(item_format.color, DisplayColumnColors.Default)
self.assertEqual(convert_func.__name__, 'convertInt')
def test_string_compare(self):
'''
FastArray currently does not support bytestring array comparison with ufuncs - numpy also prints notimplemented
However operators <=, <, ==, !=, >, >= will return the correct result (boolean array)
'''
f_arr = FA(['a', 'b', 'c'])
invalid_funcs = [
np.less_equal,
np.less,
np.equal,
np.not_equal,
np.greater,
np.greater_equal,
]
valid_func_names = [
'__ne__', '__eq__', '__ge__', '__gt__', '__le__', '__lt__'
]
correct_results = [
[False, False, False],
[True, True, True],
[True, True, True],
[False, False, False],
[True, True, True],
[False, False, False],
]
correct_dict = dict(zip(valid_func_names, correct_results))
# ufunc comparisons will not work for strings (should we implement this on our own?)
for func in invalid_funcs:
with self.assertRaises(
TypeError,
msg=f"String comparison did not raise TypeError for {func}"
):
result = func(f_arr, f_arr)
# strings need to be compared this way
for f_name in valid_func_names:
func = f_arr.__getattribute__(f_name)
result = func(f_arr)
correct = correct_dict[f_name]
for i in range(len(result)):
self.assertEqual(
result[i],
correct[i],
msg=f"String comparison failed for function {f_name}",
)
def test_NotImplemented_not_returned(self):
# See gh-5964 and gh-2091. Some of these functions are not operator
# related and were fixed for other reasons in the past.
# Cannot do string operation on
binary_funcs = [
np.power,
np.subtract,
np.multiply,
np.divide,
np.true_divide,
np.floor_divide,
np.bitwise_and,
np.bitwise_or,
np.bitwise_xor,
np.left_shift,
np.right_shift,
np.fmax,
np.fmin,
np.fmod,
np.hypot,
np.logaddexp,
np.logaddexp2,
np.logical_and,
np.logical_or,
np.logical_xor,
np.maximum,
np.minimum,
np.mod,
]
# These functions still return NotImplemented. Will be fixed in
# future.
# bad = [np.greater, np.greater_equal, np.less, np.less_equal, np.not_equal]
a = np.array('1')
a = FA(a)
b = 1
for f in binary_funcs:
with self.assertRaises(
TypeError,
msg=
f"TypeError not raised for {f} with string and int input.",
):
_ = f(a, b)
def test_replace(self):
a = FA([0, 2, 2, 3])
b = a.replace(2, 1)
assert_array_equal(b, [0, 1, 1, 3])
def test_push(self):
a = FA([5, nan, nan, 6, nan])
b = a.push()
assert_array_equal(b, [5, 5, 5, 6, 6])
def test_map(self):
a = FA([1, 1, 1, 2, 2, 2])
d = {1: 10, 2: 20}
c = a.map(d)
assert_array_equal(c, [10, 10, 10, 20, 20, 20])
def test_noncontig(self):
# no longer a warning
a = FA([1, 2, 3, 4, 5, 6])
b = FA(a[::2])
assert_array_equal(b, [1, 3, 5])
def test_inplace_int_float(self):
'''
Unlike numpy arrays, FastArray allows inplace operations between integer arrays and floating point scalars.
The datatype of the array will remain the same. However currently division is not supported.
Potential BUG: The floor division operator does not raise an error.
'''
nums = [1, 2, 3, 4, 5]
for dt in int_types[1:]: # dont include bool
arr = FA(nums, dtype=dt)
for dtf in float_types:
scalar = dtf(1)
arr += scalar
self.assertEqual(
arr.dtype,
dt,
msg=
f"Result datatype {arr.dtype} did not match original datatype {dt} after += operation",
)
arr -= scalar
self.assertEqual(
arr.dtype,
dt,
msg=
f"Result datatype {arr.dtype} did not match original datatype {dt} after -= operation",
)
arr *= scalar
self.assertEqual(
arr.dtype,
dt,
msg=
f"Result datatype {arr.dtype} did not match original datatype {dt} after *= operation",
)
arr /= scalar
self.assertEqual(
arr.dtype,
dt,
msg=
f"Result datatype {arr.dtype} did not match original datatype {dt} after /= opration",
)
arr //= scalar
self.assertEqual(
arr.dtype,
dt,
msg=
f"Result datatype {arr.dtype} did not match original datatype {dt} after //= operation",
)
for dt in int_types[1:]: # dont include bool
arr = FA(nums, dtype=dt)
for dtf in float_types:
arr2 = arr.astype(dtf)
arr += arr2
self.assertEqual(
arr.dtype,
dt,
msg=
f"Result datatype {arr.dtype} did not match original datatype {dt} after += operation",
)
arr -= arr2
self.assertEqual(
arr.dtype,
dt,
msg=
f"Result datatype {arr.dtype} did not match original datatype {dt} after -= operation",
)
arr *= arr2
self.assertEqual(
arr.dtype,
dt,
msg=
f"Result datatype {arr.dtype} did not match original datatype {dt} after *= operation",
)
arr /= arr2
self.assertEqual(
arr.dtype,
dt,
msg=
f"Result datatype {arr.dtype} did not match original datatype {dt} after /= operation",
)
arr //= arr2
self.assertEqual(
arr.dtype,
dt,
msg=
f"Result datatype {arr.dtype} did not match original datatype {dt} after //= operation",
)
def test_out(self):
a = FA(np.arange(100))
c = a
b = FA(np.arange(100.0))
np.add(b, b, out=a)
assert_array_equal(a, c)
def test_compare(self):
'''
Compares FastArray results to numpy results for binary comparison and logical ufuncs.
All results will be boolean arrays.
There is a difference between calling a numpy ufunc and using a comparison operator,
so the operators need to be checked separately.
'''
basic_types = [np.int32, np.int64, np.float32, np.float64]
numeric_types = int_types + float_types
comparison_ufuncs = [
np.less_equal,
np.less,
np.equal,
np.not_equal,
np.greater,
np.greater_equal,
]
logical_ufuncs = [np.logical_and, np.logical_xor, np.logical_or]
comparison_operators = [
'__ne__',
'__eq__',
'__ge__',
'__gt__',
'__le__',
'__lt__',
]
all_funcs = comparison_ufuncs + logical_ufuncs
for dt1 in numeric_types:
for dt2 in numeric_types:
fa_arr1 = FA(num_list, dtype=dt1)
fa_arr2 = FA(list(reversed(num_list)), dtype=dt2)
np_arr1 = np.array(num_list, dtype=dt1)
np_arr2 = np.array(list(reversed(num_list)), dtype=dt2)
for func in all_funcs:
fa_result = func(fa_arr1, fa_arr2)
np_result = func(np_arr1, np_arr2)
# check that result lengths are the same
self.assertEqual(
len(fa_result),
len(np_result),
msg=
f"Result sizes did not match for {func} with dtypes {dt1} {dt2}",
)
# compare each result item
arr_size = len(fa_result)
for i in range(arr_size):
self.assertEqual(
fa_result[i],
np_result[i],
msg=
f"Comparison result did not match for {func} with dtypes {dt1} {dt2}",
)
for f_name in comparison_operators:
fa_func = fa_arr1.__getattribute__(f_name)
np_func = np_arr1.__getattribute__(f_name)
fa_result = fa_func(fa_arr2)
np_result = np_func(np_arr2)
# check that result lengths are the same
self.assertEqual(
len(fa_result),
len(np_result),
msg=
f"Result sizes did not match for operator {f_name} with dtypes {dt1} {dt2}",
)
# compare each result item
arr_size = len(fa_result)
for i in range(arr_size):
self.assertEqual(
fa_result[i],
np_result[i],
msg=
f"Comparison operator {f_name} failed with dtypes {dt1} {dt2}",
)
def test_empty_ufuncs(self):
n = np.array([])
f = FA([])
pass
def test_cut(self):
c = cut(arange(10), 3)
self.assertTrue(sum(c._np - FA([1, 1, 1, 1, 2, 2, 2, 3, 3, 3])) == 0)
c = cut(arange(10.0), 3)
self.assertTrue(sum(c._np - FA([1, 1, 1, 1, 2, 2, 2, 3, 3, 3])) == 0)
c = cut(arange(11), 3)
self.assertTrue(
sum(c._np - FA([1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3])) == 0)
c = cut(FA([2, 4, 6, 8, 10]), FA([0, 2, 4, 6, 8, 10]))
self.assertTrue(sum(c._np - FA([1, 2, 3, 4, 5])) == 0)
c = cut(
FA([2, 4, 6, 8, 10]),
FA([0, 2, 4, 6, 8, 10]),
labels=['a', 'b', 'c', 'd', 'e'],
)
self.assertTrue(sum(c._np - FA([1, 2, 3, 4, 5])) == 0)
a = np.array([1, 7, 5, 4, 6, 3])
l = FA([b'1.0->3.0', b'3.0->5.0', b'5.0->7.0'])
c = cut(a, 3)
self.assertIsInstance(c, Categorical)
self.assertTrue(sum(c._np - FA([1, 3, 2, 2, 3, 1])) == 0)
self.assertTrue((c.category_array == l).all())
c = cut(a, 3, labels=True)
self.assertIsInstance(c, Categorical)
self.assertTrue(sum(c._np - FA([1, 3, 2, 2, 3, 1])) == 0)
self.assertTrue((c.category_array == l).all())
c = cut(a, 3, labels=None)
self.assertIsInstance(c, Categorical)
self.assertTrue(sum(c._np - FA([1, 3, 2, 2, 3, 1])) == 0)
self.assertTrue((c.category_array == l).all())
c = cut(a, 3, labels=False)
self.assertIsInstance(c, FastArray)
self.assertTrue(sum(c._np - FA([1, 3, 2, 2, 3, 1])) == 0)
c, b = cut(a, 3, retbins=True)
self.assertIsInstance(c, Categorical)
self.assertIsInstance(b, np.ndarray)
self.assertTrue(sum(c._np - FA([1, 3, 2, 2, 3, 1])) == 0)
self.assertTrue((c.category_array == l).all())
self.assertTrue(sum(b - FA([1.0, 3.0, 5.0, 7.0])) == 0)
l = ["bad", "medium", "good"]
c = cut(a, 3, labels=l)
self.assertIsInstance(c, Categorical)
self.assertTrue(sum(c._np - FA([1, 3, 2, 2, 3, 1])) == 0)
self.assertTrue((c.category_array == l).all())
# contiguous test
x = arange(4).reshape(2, 2)
knots = [-0.5, 0.5, 1.5, 2.5, 3.5]
c = cut(x[:, 1], knots)
l = FastArray([b'-0.5->0.5', b'0.5->1.5', b'1.5->2.5', b'2.5->3.5'])
self.assertTrue((c.category_array == l).all())
# inf upcast test
x = np.array([0, 1, 10, 100, 5])
knots = [-np.inf, 2, 11, 50, np.inf]
c = cut(x, knots)
self.assertTrue((c._fa == FA([1, 1, 2, 4, 2])).all())
