def append(self, data, indices=None):
if data is None:
return
l = len(data)
is_array = isinstance(data, (np.ndarray, list, BaseColumn))
if indices is not None and len(indices) != l:
raise ValueError('Bad index length (%d/%d)', len(indices), l)
indices = self._allocate(len(data), indices)
if is_array:
indices = indices_to_slice(indices)
self[indices] = data[0:l]
else:
for i in range(l):
self[indices[i]] = data[i]
def id_to_index(self, loc, as_slice=True):
if self._is_identity:
if isinstance(loc, slice): # slices are inclusive
if loc.stop != None:
return slice(loc.start, loc.stop + 1, loc.step)
if is_none_alike(loc):
loc = slice(0, self.size, 1)
elif isinstance(loc, integer_types):
if loc < 0:
loc += self._last_id
return loc
if self._ids_dict is None:
self._update_ids_dict()
if is_none_alike(loc): # return everything
ret = bitmap(range(0, self.size))
if self._freelist:
ret -= self._freelist
if as_slice:
ret = ret.to_slice_maybe()
return ret
if isinstance(loc, np.ndarray) and loc.dtype == np.int:
ret = self._ids_dict.get_items(loc)
elif isinstance(loc, integer_types):
if loc < 0:
loc = self._last_id + loc
return self._ids_dict[loc]
elif isinstance(loc, Iterable):
try:
count = len(loc)
# pylint: disable=bare-except
except:
count = -1
ret = np.fromiter(loc, dtype=np.int64, count=count)
ret = self._ids_dict.get_items(ret)
elif isinstance(loc, slice):
loc_stop = self.last_id if loc.stop is None else loc.stop + 1
ret = np.array(range(loc.start, loc_stop, loc.step or 1),
dtype=np.int64)
ret = self._ids_dict.get_items(ret)
else:
raise ValueError('id_to_index not implemented for id "%s"' % loc)
return indices_to_slice(ret) if as_slice else ret
def test_indices_to_slice(self):
s = indices_to_slice([])
self.assertEqual(s, slice(0, 0))
s = indices_to_slice([0, 1, 2, 3])
self.assertEqual(s, slice(0, 4))
# not sure the following are desirable
#s = indices_to_slice([0, 1, 1, 2, 3])
#self.assertEqual(s, slice(0,4))
s = indices_to_slice([1, 2, 3])
self.assertEqual(s, slice(1, 4))
s = indices_to_slice([1, 2, 3, 5])
self.assertEqual(s, [1, 2, 3, 5])
s = indices_to_slice(np.array([0, 1, 2, 3]))
self.assertEqual(s, slice(0, 4))
s = indices_to_slice(np.array([1, 2, 3, 4]))
self.assertEqual(s, slice(1, 5))
def append(self, data, indices=None):
"""
Append Table-like data to the Table.
The data has to be compatible. It can be from multiple sources [more details needed].
"""
if data is None:
return
data = self.parse_data(data, indices)
dshape = dshape_extract(data)
if not dshape_compatible(dshape, self.dshape):
raise ValueError(
"{shape} incompatible data shape in append".format(
shape=str(dshape)))
length = -1
all_arrays = True
for colname in self:
fromcol = data[colname]
if length is -1:
length = len(fromcol)
elif length != len(fromcol):
raise ValueError('Cannot append ragged values')
all_arrays |= isinstance(fromcol, np.ndarray)
if length == 0:
return
if indices is not None and len(indices) != length:
raise ValueError('Bad index length (%d/%d)', len(indices), length)
indices = self._allocate(length, indices)
if all_arrays:
indices = indices_to_slice(indices)
for colname in self:
tocol = self._column(colname)
fromcol = data[colname]
tocol[indices] = fromcol[0:length]
else:
for colname in self:
tocol = self._column(colname)
fromcol = data[colname]
for i in range(length):
tocol[indices[i]] = fromcol[i]
def id_to_index(self, loc, as_slice=True):
if self._ids_dict is None:
self._update_ids_dict()
if isinstance(loc, np.ndarray) and loc.dtype == np.int64:
ret = self._ids_dict.get_items(loc)
elif isinstance(loc, integer_types):
if loc < 0:
loc = self._last_id + loc
return self._ids_dict[loc]
elif isinstance(loc, Iterable):
try:
count = len(loc)
# pylint: disable=bare-except
except:
count = -1
ret = np.fromiter(loc, dtype=np.int64, count=count)
ret = self._ids_dict.get_items(ret)
elif isinstance(loc, slice):
ret = np.array(range(loc.start, loc.stop + 1, loc.step or 1),
dtype=np.int64)
ret = self._ids_dict.get_items(ret)
else:
raise ValueError('id_to_index not implemented for id "%s"' % loc)
return indices_to_slice(ret) if as_slice else ret
def eval(self,
expr,
inplace=False,
name=None,
result_object=None,
user_dict=None,
as_slice=True):
"""Evaluate the ``expr`` on columns and return the result.
Args:
inplace: boolean, optional
Apply the changes in place
name: string
used when a new table/view is created, otherwise ignored
result_object: string
Posible values for result_object: {'raw_numexpr', 'index', 'view', 'table'}
When expr is conditional.
Note: a result as 'view' is not guaranteed: it may be 'table' when the calculated
index is not sliceable
- 'table' or None when expr is an assignment
Default values for result_object :
- 'indices' when expr is conditional
- NA i.e. always None when inplace=True, otherwise a new table is returned
"""
if inplace and result_object:
raise ValueError(
"'inplace' and 'result_object' options are not compatible")
if user_dict is None:
context = {key: self[key].values for key in self.columns}
else:
context = user_dict
is_assign = False
try:
res = ne.evaluate(expr, local_dict=context)
if result_object is None:
result_object = 'index'
except SyntaxError as err:
# maybe an assignment ?
try:
l_col, r_expr = expr.split('=', 1)
l_col = l_col.strip()
if l_col not in self.columns:
raise err
res = ne.evaluate(r_expr.strip(), local_dict=context)
is_assign = True
except:
raise err
if result_object is not None and result_object != 'table':
raise ValueError("result_object={} is not valid when expr "
"is an assignment".format(result_object))
else:
if result_object not in ('raw_numexpr', 'index', 'view', 'table'):
raise ValueError(
"result_object={} is not valid".format(result_object))
if is_assign:
if inplace:
self[l_col] = res
return
# then not inplace ...
def cval(key):
return res if key == l_col else self[key].values
data = [(cname, cval(cname)) for cname in self.columns]
return Table(name=name, data=OrderedDict(data), indices=self.index)
# not an assign ...
if res.dtype != 'bool':
raise ValueError('expr must be an assignment '
'or a conditional expr.!')
if inplace:
raise ValueError('inplace eval of conditional expr '
'not implemented!')
if result_object == 'raw_numexpr':
return res
indices = np.where(res)[0]
if not as_slice and result_object == 'index':
return indices
ix_slice = indices_to_slice(indices)
if result_object == 'index':
return ix_slice
if result_object == 'view':
return self.iloc[ix_slice, :]
# as a new table ...
data = [(cname, self[cname].values[indices]) for cname in self.columns]
return Table(name=name,
data=OrderedDict(data),
indices=self._ids.values[indices])