def apply_index(self, index_to_apply, src, dest=None):
"""
Apply a index to an a src field. The indexed field is written to dest if it set,
and returned from the function call. If the field is an IndexedStringField, the
indices and values are returned separately.
:param index_to_apply: the index to be applied to the source field
:param src: the field to be index
:param dest: optional - a field to write the indexed data to
:return: the indexed values
"""
index_to_apply_ = val.array_from_parameter(self, 'index_to_apply',
index_to_apply)
writer_ = None
if dest is not None:
writer_ = val.field_from_parameter(self, 'writer', dest)
if isinstance(src, fld.IndexedStringField):
src_ = val.field_from_parameter(self, 'reader', src)
dest_indices, dest_values =\
ops.apply_indices_to_index_values(index_to_apply_,
src_.indices[:], src_.values[:])
if writer_ is not None:
writer_.indices.write(dest_indices)
writer_.values.write(dest_values)
return dest_indices, dest_values
else:
reader_ = val.array_from_parameter(self, 'reader', src)
result = reader_[index_to_apply]
if writer_:
writer_.data.write(result)
return result
def _streaming_map_fields(self, field_map, field_sources, field_sinks):
# field map must be a field
# field sources must be fields
# field sinks must be fields
# field sources and sinks must be the same length
map_ = val.field_from_parameter(self, 'field_map', field_map)
for src, snk in zip(field_sources, field_sinks):
src_ = val.field_from_parameter(self, 'field_sources', src)
snk_ = val.field_from_parameter(self, 'field_sinks', snk)
ops.ordered_map_valid_stream(src_, map_, snk_)
def _map_fields(self, field_map, field_sources, field_sinks):
rtn_sinks = None
if field_sinks is None:
left_sinks = list()
for src in field_sources:
src_ = val.array_from_parameter(self, 'left_field_sources',
src)
snk_ = ops.map_valid(src_, field_map)
left_sinks.append(snk_)
rtn_sinks = left_sinks
elif val.is_field_parameter(field_sinks[0]):
# groups or fields
for src, snk in zip(field_sources, field_sinks):
src_ = val.array_from_parameter(self, 'left_field_sources',
src)
snk_ = ops.map_valid(src_, field_map)
snk = val.field_from_parameter(self, 'left_field_sinks', snk)
snk.data.write(snk_)
else:
# raw arrays
for src, snk in zip(field_sources, field_sinks):
src_ = val.array_from_parameter(self, 'left_field_sources',
src)
snk_ = val.array_from_parameter(self, 'left_field_sinks', snk)
ops.map_valid(src_, field_map, snk_)
return None if rtn_sinks is None else tuple(rtn_sinks)
def _apply_spans_no_src(self, predicate, spans, dest=None):
assert (dest is None or isinstance(dest, fld.Field))
if dest is not None:
dest_f = val.field_from_parameter(self, 'dest', dest)
results = np.zeros(len(spans) - 1, dtype=dest_f.data.dtype)
predicate(spans, results)
dest_f.data.write(results)
return results
else:
results = np.zeros(len(spans) - 1, dtype='int64')
predicate(spans, results)
return results
def _aggregate_impl(self, predicate, index, src=None, dest=None):
index_ = val.raw_array_from_parameter(self, "index", index)
dest_field = None
if dest is not None:
dest_field = val.field_from_parameter(self, "dest", dest)
fkey_index_spans = self.get_spans(field=index)
# execute the predicate (note that not every predicate requires a reader)
results = predicate(fkey_index_spans, src, dest_field)
return dest if dest is not None else results
def _apply_spans_src(self, predicate, spans, src, dest=None):
assert (dest is None or isinstance(dest, fld.Field))
src_ = val.array_from_parameter(self, 'src', src)
if len(src) != spans[-1]:
error_msg = (
"'src' (length {}) must be one element shorter than 'spans' "
"(length {})")
raise ValueError(error_msg.format(len(src_), len(spans)))
if dest is not None:
dest_f = val.field_from_parameter(self, 'dest', dest)
results = np.zeros(len(spans) - 1, dtype=dest_f.data.dtype)
predicate(spans, src_, results)
dest_f.data.write(results)
return results
else:
results = np.zeros(len(spans) - 1, dtype=src_.dtype)
predicate(spans, src_, results)
return results