def _concat_compat(to_concat, axis=0):
"""
provide concatenation of an datetimelike array of arrays each of which is a single
M8[ns], datetimet64[ns, tz] or m8[ns] dtype
Parameters
----------
to_concat : array of arrays
axis : axis to provide concatenation
Returns
-------
a single array, preserving the combined dtypes
"""
def convert_to_pydatetime(x, axis):
# coerce to an object dtype
if x.dtype == _NS_DTYPE:
if hasattr(x, 'tz'):
x = x.asobject
shape = x.shape
x = tslib.ints_to_pydatetime(x.view(np.int64).ravel())
x = x.reshape(shape)
elif x.dtype == _TD_DTYPE:
shape = x.shape
x = tslib.ints_to_pytimedelta(x.view(np.int64).ravel())
x = x.reshape(shape)
return x
typs = get_dtype_kinds(to_concat)
# datetimetz
if 'datetimetz' in typs:
# we require ALL of the same tz for datetimetz
tzs = set([getattr(x, 'tz', None) for x in to_concat]) - set([None])
if len(tzs) == 1:
return DatetimeIndex(np.concatenate(
[x.tz_localize(None).asi8 for x in to_concat]),
tz=list(tzs)[0])
# single dtype
if len(typs) == 1:
if not len(typs - set(['datetime'])):
new_values = np.concatenate([x.view(np.int64) for x in to_concat],
axis=axis)
return new_values.view(_NS_DTYPE)
elif not len(typs - set(['timedelta'])):
new_values = np.concatenate([x.view(np.int64) for x in to_concat],
axis=axis)
return new_values.view(_TD_DTYPE)
# need to coerce to object
to_concat = [convert_to_pydatetime(x, axis) for x in to_concat]
return np.concatenate(to_concat, axis=axis)
def _concat_compat(to_concat, axis=0):
"""
provide concatenation of an datetimelike array of arrays each of which is a single
M8[ns], datetimet64[ns, tz] or m8[ns] dtype
Parameters
----------
to_concat : array of arrays
axis : axis to provide concatenation
Returns
-------
a single array, preserving the combined dtypes
"""
def convert_to_pydatetime(x, axis):
# coerce to an object dtype
if x.dtype == _NS_DTYPE:
if hasattr(x, 'tz'):
x = x.asobject
shape = x.shape
x = tslib.ints_to_pydatetime(x.view(np.int64).ravel())
x = x.reshape(shape)
elif x.dtype == _TD_DTYPE:
shape = x.shape
x = tslib.ints_to_pytimedelta(x.view(np.int64).ravel())
x = x.reshape(shape)
return x
typs = get_dtype_kinds(to_concat)
# datetimetz
if 'datetimetz' in typs:
# we require ALL of the same tz for datetimetz
tzs = set([ getattr(x,'tz',None) for x in to_concat ])-set([None])
if len(tzs) == 1:
return DatetimeIndex(np.concatenate([ x.tz_localize(None).asi8 for x in to_concat ]), tz=list(tzs)[0])
# single dtype
if len(typs) == 1:
if not len(typs-set(['datetime'])):
new_values = np.concatenate([x.view(np.int64) for x in to_concat],
axis=axis)
return new_values.view(_NS_DTYPE)
elif not len(typs-set(['timedelta'])):
new_values = np.concatenate([x.view(np.int64) for x in to_concat],
axis=axis)
return new_values.view(_TD_DTYPE)
# need to coerce to object
to_concat = [convert_to_pydatetime(x, axis) for x in to_concat]
return np.concatenate(to_concat,axis=axis)
def _concat_compat(to_concat, axis=0):
"""
provide concatenation of an datetimelike array of arrays each of which is a single
M8[ns], or m8[ns] dtype
Parameters
----------
to_concat : array of arrays
axis : axis to provide concatenation
Returns
-------
a single array, preserving the combined dtypes
"""
def convert_to_pydatetime(x, axis):
# coerce to an object dtype
if x.dtype == _NS_DTYPE:
shape = x.shape
x = tslib.ints_to_pydatetime(x.view(np.int64).ravel())
x = x.reshape(shape)
elif x.dtype == _TD_DTYPE:
shape = x.shape
x = tslib.ints_to_pytimedelta(x.view(np.int64).ravel())
x = x.reshape(shape)
return x
typs = get_dtype_kinds(to_concat)
# single dtype
if len(typs) == 1:
if not len(typs-set(['datetime'])):
new_values = np.concatenate([x.view(np.int64) for x in to_concat],
axis=axis)
return new_values.view(_NS_DTYPE)
elif not len(typs-set(['timedelta'])):
new_values = np.concatenate([x.view(np.int64) for x in to_concat],
axis=axis)
return new_values.view(_TD_DTYPE)
# need to coerce to object
to_concat = [convert_to_pydatetime(x, axis) for x in to_concat]
return np.concatenate(to_concat,axis=axis)
def _concat_compat(to_concat, axis=0):
"""
provide concatenation of an datetimelike array of arrays each of which is a single
M8[ns], or m8[ns] dtype
Parameters
----------
to_concat : array of arrays
axis : axis to provide concatenation
Returns
-------
a single array, preserving the combined dtypes
"""
def convert_to_pydatetime(x, axis):
# coerce to an object dtype
if x.dtype == _NS_DTYPE:
shape = x.shape
x = tslib.ints_to_pydatetime(x.view(np.int64).ravel())
x = x.reshape(shape)
elif x.dtype == _TD_DTYPE:
shape = x.shape
x = tslib.ints_to_pytimedelta(x.view(np.int64).ravel())
x = x.reshape(shape)
return x
typs = get_dtype_kinds(to_concat)
# single dtype
if len(typs) == 1:
if not len(typs-set(['datetime'])):
new_values = np.concatenate([x.view(np.int64) for x in to_concat],
axis=axis)
return new_values.view(_NS_DTYPE)
elif not len(typs-set(['timedelta'])):
new_values = np.concatenate([x.view(np.int64) for x in to_concat],
axis=axis)
return new_values.view(_TD_DTYPE)
# need to coerce to object
to_concat = [convert_to_pydatetime(x, axis) for x in to_concat]
return np.concatenate(to_concat,axis=axis)
def _concat_compat(to_concat, axis=0):
"""
provide concatenation of an sparse/dense array of arrays each of which is a
single dtype
Parameters
----------
to_concat : array of arrays
axis : axis to provide concatenation
Returns
-------
a single array, preserving the combined dtypes
"""
def convert_sparse(x, axis):
# coerce to native type
if isinstance(x, SparseArray):
x = x.get_values()
x = x.ravel()
if axis > 0:
x = np.atleast_2d(x)
return x
typs = com.get_dtype_kinds(to_concat)
# we have more than one type here, so densify and regular concat
to_concat = [convert_sparse(x, axis) for x in to_concat]
result = np.concatenate(to_concat, axis=axis)
if not len(typs - set(['sparse', 'f', 'i'])):
# we can remain sparse
result = SparseArray(result.ravel())
else:
# coerce to object if needed
result = result.astype('object')
return result
def _concat_sparse(to_concat, axis=0, typs=None):
"""
provide concatenation of an sparse/dense array of arrays each of which is a
single dtype
Parameters
----------
to_concat : array of arrays
axis : axis to provide concatenation
typs : set of to_concat dtypes
Returns
-------
a single array, preserving the combined dtypes
"""
from pandas.sparse.array import SparseArray, _make_index
def convert_sparse(x, axis):
# coerce to native type
if isinstance(x, SparseArray):
x = x.get_values()
x = x.ravel()
if axis > 0:
x = np.atleast_2d(x)
return x
if typs is None:
typs = com.get_dtype_kinds(to_concat)
if len(typs) == 1:
# concat input as it is if all inputs are sparse
# and have the same fill_value
fill_values = set(c.fill_value for c in to_concat)
if len(fill_values) == 1:
sp_values = [c.sp_values for c in to_concat]
indexes = [c.sp_index.to_int_index() for c in to_concat]
indices = []
loc = 0
for idx in indexes:
indices.append(idx.indices + loc)
loc += idx.length
sp_values = np.concatenate(sp_values)
indices = np.concatenate(indices)
sp_index = _make_index(loc, indices, kind=to_concat[0].sp_index)
return SparseArray(sp_values,
sparse_index=sp_index,
fill_value=to_concat[0].fill_value)
# input may be sparse / dense mixed and may have different fill_value
# input must contain sparse at least 1
sparses = [c for c in to_concat if com.is_sparse(c)]
fill_values = [c.fill_value for c in sparses]
sp_indexes = [c.sp_index for c in sparses]
# densify and regular concat
to_concat = [convert_sparse(x, axis) for x in to_concat]
result = np.concatenate(to_concat, axis=axis)
if not len(typs - set(['sparse', 'f', 'i'])):
# sparsify if inputs are sparse and dense numerics
# first sparse input's fill_value and SparseIndex is used
result = SparseArray(result.ravel(),
fill_value=fill_values[0],
kind=sp_indexes[0])
else:
# coerce to object if needed
result = result.astype('object')
return result
def _concat_sparse(to_concat, axis=0, typs=None):
"""
provide concatenation of an sparse/dense array of arrays each of which is a
single dtype
Parameters
----------
to_concat : array of arrays
axis : axis to provide concatenation
typs : set of to_concat dtypes
Returns
-------
a single array, preserving the combined dtypes
"""
from pandas.sparse.array import SparseArray, _make_index
def convert_sparse(x, axis):
# coerce to native type
if isinstance(x, SparseArray):
x = x.get_values()
x = x.ravel()
if axis > 0:
x = np.atleast_2d(x)
return x
if typs is None:
typs = com.get_dtype_kinds(to_concat)
if len(typs) == 1:
# concat input as it is if all inputs are sparse
# and have the same fill_value
fill_values = set(c.fill_value for c in to_concat)
if len(fill_values) == 1:
sp_values = [c.sp_values for c in to_concat]
indexes = [c.sp_index.to_int_index() for c in to_concat]
indices = []
loc = 0
for idx in indexes:
indices.append(idx.indices + loc)
loc += idx.length
sp_values = np.concatenate(sp_values)
indices = np.concatenate(indices)
sp_index = _make_index(loc, indices, kind=to_concat[0].sp_index)
return SparseArray(sp_values, sparse_index=sp_index,
fill_value=to_concat[0].fill_value)
# input may be sparse / dense mixed and may have different fill_value
# input must contain sparse at least 1
sparses = [c for c in to_concat if com.is_sparse(c)]
fill_values = [c.fill_value for c in sparses]
sp_indexes = [c.sp_index for c in sparses]
# densify and regular concat
to_concat = [convert_sparse(x, axis) for x in to_concat]
result = np.concatenate(to_concat, axis=axis)
if not len(typs - set(['sparse', 'f', 'i'])):
# sparsify if inputs are sparse and dense numerics
# first sparse input's fill_value and SparseIndex is used
result = SparseArray(result.ravel(), fill_value=fill_values[0],
kind=sp_indexes[0])
else:
# coerce to object if needed
result = result.astype('object')
return result
def _concat_compat(to_concat, axis=0, typs=None):
"""
provide concatenation of an datetimelike array of arrays each of which is a
single M8[ns], datetimet64[ns, tz] or m8[ns] dtype
Parameters
----------
to_concat : array of arrays
axis : axis to provide concatenation
Returns
-------
a single array, preserving the combined dtypes
"""
def convert_to_pydatetime(x, axis):
# coerce to an object dtype
# if dtype is of datetimetz or timezone
if x.dtype.kind == _NS_DTYPE.kind:
if getattr(x, 'tz', None) is not None:
x = x.asobject
else:
shape = x.shape
x = tslib.ints_to_pydatetime(x.view(np.int64).ravel())
x = x.reshape(shape)
elif x.dtype == _TD_DTYPE:
shape = x.shape
x = tslib.ints_to_pytimedelta(x.view(np.int64).ravel())
x = x.reshape(shape)
return x
if typs is None:
typs = get_dtype_kinds(to_concat)
# must be single dtype
if len(typs) == 1:
if 'datetimetz' in typs:
# datetime with no tz should be stored as "datetime" in typs,
# thus no need to care
# we require ALL of the same tz for datetimetz
tzs = set([x.tz for x in to_concat])
if len(tzs) == 1:
return DatetimeIndex(np.concatenate([x.tz_localize(None).asi8
for x in to_concat]),
tz=list(tzs)[0])
elif 'datetime' in typs:
new_values = np.concatenate([x.view(np.int64) for x in to_concat],
axis=axis)
return new_values.view(_NS_DTYPE)
elif 'timedelta' in typs:
new_values = np.concatenate([x.view(np.int64) for x in to_concat],
axis=axis)
return new_values.view(_TD_DTYPE)
# need to coerce to object
to_concat = [convert_to_pydatetime(x, axis) for x in to_concat]
return np.concatenate(to_concat, axis=axis)
def _concat_compat(to_concat, axis=0, typs=None):
"""
provide concatenation of an datetimelike array of arrays each of which is a
single M8[ns], datetimet64[ns, tz] or m8[ns] dtype
Parameters
----------
to_concat : array of arrays
axis : axis to provide concatenation
Returns
-------
a single array, preserving the combined dtypes
"""
def convert_to_pydatetime(x, axis):
# coerce to an object dtype
# if dtype is of datetimetz or timezone
if x.dtype.kind == _NS_DTYPE.kind:
if getattr(x, 'tz', None) is not None:
x = x.asobject.values
else:
shape = x.shape
x = tslib.ints_to_pydatetime(x.view(np.int64).ravel())
x = x.reshape(shape)
elif x.dtype == _TD_DTYPE:
shape = x.shape
x = tslib.ints_to_pytimedelta(x.view(np.int64).ravel())
x = x.reshape(shape)
if axis == 1:
x = np.atleast_2d(x)
return x
if typs is None:
typs = get_dtype_kinds(to_concat)
# must be single dtype
if len(typs) == 1:
if 'datetimetz' in typs:
# datetime with no tz should be stored as "datetime" in typs,
# thus no need to care
# we require ALL of the same tz for datetimetz
tzs = set([x.tz for x in to_concat])
if len(tzs) == 1:
return DatetimeIndex(np.concatenate([x.tz_localize(None).asi8
for x in to_concat]),
tz=list(tzs)[0])
elif 'datetime' in typs:
new_values = np.concatenate([x.view(np.int64) for x in to_concat],
axis=axis)
return new_values.view(_NS_DTYPE)
elif 'timedelta' in typs:
new_values = np.concatenate([x.view(np.int64) for x in to_concat],
axis=axis)
return new_values.view(_TD_DTYPE)
# need to coerce to object
to_concat = [convert_to_pydatetime(x, axis) for x in to_concat]
return np.concatenate(to_concat, axis=axis)
