def _aggregate_series_pure_python(self, obj, func):
group_index, _, ngroups = self.group_info
counts = np.zeros(ngroups, dtype=int)
result = None
group_index, _, ngroups = self.group_info
for label, group in _generate_groups(obj,
group_index,
ngroups,
axis=self.axis):
res = func(group)
if result is None:
try:
assert (not isinstance(res, np.ndarray))
assert (not isinstance(res, list))
result = np.empty(ngroups, dtype='O')
except Exception:
raise ValueError('function does not reduce')
counts[label] = group.shape[0]
result[label] = res
result = lib.maybe_convert_objects(result, try_float=0)
return result, counts
def test_convert_objects_ints():
# test that we can detect many kinds of integers
dtypes = ['i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8']
for dtype_str in dtypes:
arr = np.array(list(np.arange(20, dtype=dtype_str)), dtype='O')
assert (arr[0].dtype == np.dtype(dtype_str))
result = lib.maybe_convert_objects(arr)
assert (issubclass(result.dtype.type, np.integer))
def test_convert_objects_ints():
# test that we can detect many kinds of integers
dtypes = ['i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8']
for dtype_str in dtypes:
arr = np.array(list(np.arange(20, dtype=dtype_str)), dtype='O')
assert(arr[0].dtype == np.dtype(dtype_str))
result = lib.maybe_convert_objects(arr)
assert(issubclass(result.dtype.type, np.integer))
def _aggregate_series_pure_python(self, obj, func, group_index, ngroups):
counts = np.zeros(ngroups, dtype=int)
result = None
for label, group in self._generate_groups(obj, group_index, ngroups):
res = func(group)
if result is None:
try:
assert not isinstance(res, np.ndarray)
assert not (isinstance(res, list) and len(res) == len(self.dummy))
result = np.empty(ngroups, dtype="O")
except Exception:
raise ValueError("function does not reduce")
counts[label] = group.shape[0]
result[label] = res
result = lib.maybe_convert_objects(result, try_float=0)
return result, counts
def unique1d(values):
"""
Hash table-based unique
"""
if issubclass(values.dtype.type, np.floating):
if values.dtype != np.float64:
values = values.astype(np.float64)
table = lib.Float64HashTable(len(values))
uniques = np.array(table.unique(values), dtype=np.float64)
elif issubclass(values.dtype.type, np.integer):
if values.dtype != np.int64:
values = values.astype(np.int64)
table = lib.Int64HashTable(len(values))
uniques = np.array(table.unique(values), dtype=np.int64)
else:
if not values.dtype == np.object_:
values = values.astype(np.object_)
table = lib.PyObjectHashTable(len(values))
uniques = lib.list_to_object_array(table.unique(values))
uniques = lib.maybe_convert_objects(uniques)
return uniques
def _aggregate_series_pure_python(self, obj, func, ngroups):
counts = np.zeros(ngroups, dtype=int)
result = None
for label, group in self._generator_factory(obj):
if group is None:
continue
res = func(group)
if result is None:
try:
assert(not isinstance(res, np.ndarray))
assert(not (isinstance(res, list) and
len(res) == len(self.dummy)))
result = np.empty(ngroups, dtype='O')
except Exception:
raise ValueError('function does not reduce')
counts[label] = group.shape[0]
result[label] = res
result = lib.maybe_convert_objects(result)
return result, counts
def map_iter_args(arr, f, otherargs, n_otherargs, required, n_results):
'''
Substitute for np.vectorize with pandas-friendly dtype inference
Parameters
----------
arr : ndarray
f : function
Returns
-------
mapped : ndarray
'''
n = len(arr)
result = np.empty((n, n_results), dtype=object)
for i, val in enumerate(arr):
args = otherargs[i]
if notnull(val) and all(notnull(args[r]) for r in required):
result[i] = f(val, *args)
else:
result[i] = [np.nan] * n_results
return [lib.maybe_convert_objects(col, try_float=0) for col in result.T]
def map_iter_args(arr, f, otherargs, n_otherargs, required, n_results):
'''
Substitute for np.vectorize with pandas-friendly dtype inference
Parameters
----------
arr : ndarray
f : function
Returns
-------
mapped : ndarray
'''
n = len(arr)
result = np.empty((n, n_results), dtype=object)
for i, val in enumerate(arr):
args = otherargs[i]
if notnull(val) and all(notnull(args[r]) for r in required):
result[i] = f(val, *args)
else:
result[i] = [np.nan] * n_results
return [lib.maybe_convert_objects(col, try_float=0) for col in result.T]
def _aggregate_series_pure_python(self, obj, func):
group_index, _, ngroups = self.group_info
counts = np.zeros(ngroups, dtype=int)
result = None
group_index, _, ngroups = self.group_info
for label, group in _generate_groups(obj, group_index, ngroups,
axis=self.axis):
res = func(group)
if result is None:
try:
assert(not isinstance(res, np.ndarray))
assert(not isinstance(res, list))
result = np.empty(ngroups, dtype='O')
except Exception:
raise ValueError('function does not reduce')
counts[label] = group.shape[0]
result[label] = res
result = lib.maybe_convert_objects(result, try_float=0)
return result, counts
def test_convert_objects():
arr = np.array(['a', 'b', np.nan, np.nan, 'd', 'e', 'f'], dtype='O')
result = lib.maybe_convert_objects(arr)
assert (result.dtype == np.object_)
def test_convert_objects_complex_number():
for dtype in np.sctypes['complex']:
arr = np.array(list(1j * np.arange(20, dtype=dtype)), dtype='O')
assert(arr[0].dtype == np.dtype(dtype))
result = lib.maybe_convert_objects(arr)
assert(issubclass(result.dtype.type, np.complexfloating))
def test_convert_objects():
arr = np.array(['a', 'b', nan, nan, 'd', 'e', 'f'], dtype='O')
result = lib.maybe_convert_objects(arr)
assert(result.dtype == np.object_)