def __init__(self, shmem):
AggregationTask.__init__(self, shmem)
reader = wire.PackedMessageReader(shmem)
# TODO: may wish to merge more than 2 at a time?
# Unpack header
self.left_inst = compat.pickle_load(reader.string())
self.right_inst = compat.pickle_load(reader.string())
def test_merge(self):
klass = self._get_mean_uda()
lcol = self.col_fragments[0]
rcol = self.col_fragments[1]
left = self._update(klass, [lcol])
right = self._update(klass, [rcol])
task, mm = self._make_merge_task(left, right)
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = compat.pickle_load(reader.string())
larr = lcol.to_numpy_for_pandas()
rarr = rcol.to_numpy_for_pandas()
assert larr is not None
ex_total = (pd.Series(larr).sum() + pd.Series(rarr).sum())
assert result.total == ex_total
def test_merge(self):
klass = self._get_mean_uda()
lcol = self.col_fragments[0]
rcol = self.col_fragments[1]
left = self._update(klass, [lcol])
right = self._update(klass, [rcol])
task, mm = self._make_merge_task(left, right)
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = compat.pickle_load(reader.string())
larr = lcol.to_numpy_for_pandas()
rarr = rcol.to_numpy_for_pandas()
assert larr is not None
ex_total = (pd.Series(larr).sum() + pd.Series(rarr).sum())
assert result.total == ex_total
def run(self):
if self.prior_state is not None:
agg_inst = self.prior_state
else:
klass = compat.pickle_load(self.agg_class_pickled)
agg_inst = klass()
args = self._deserialize_args()
agg_inst.update(*args)
self._write_response(agg_inst)
def test_update(self):
klass = self._get_mean_uda()
col = self.col_fragments[0]
task, mm = self._make_update_task(klass, [col])
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = compat.pickle_load(reader.string())
ex_total = pd.Series(col.to_numpy_for_pandas()).sum()
assert result.total == ex_total
# Test with prior state
col = self.col_fragments[1]
task, mm = self._make_update_task(klass, [col], prior_state=result)
# Executor's turn again
self.lock.release()
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = compat.pickle_load(reader.string())
ex_total += pd.Series(col.to_numpy_for_pandas()).sum()
# pandas will yield 0 on None input strangely
assert ex_total != 0
assert result.total == ex_total
def test_update(self):
klass = self._get_mean_uda()
col = self.col_fragments[0]
task, mm = self._make_update_task(klass, [col])
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = compat.pickle_load(reader.string())
ex_total = pd.Series(col.to_numpy_for_pandas()).sum()
assert result.total == ex_total
# Test with prior state
col = self.col_fragments[1]
task, mm = self._make_update_task(klass, [col], prior_state=result)
# Executor's turn again
self.lock.release()
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = compat.pickle_load(reader.string())
ex_total += pd.Series(col.to_numpy_for_pandas()).sum()
# pandas will yield 0 on None input strangely
assert ex_total != 0
assert result.total == ex_total
def _read_header(self):
reader = wire.PackedMessageReader(self.shmem)
# Unpack header
self.agg_class_pickled = reader.string()
has_prior_state = reader.uint8() != 0
if has_prior_state:
self.prior_state = compat.pickle_load(reader.string())
else:
self.prior_state = None
def test_finalize(self):
klass = self._get_mean_uda()
col = self.col_fragments[0]
result = self._update(klass, [col])
task, mm = self._make_finalize_task(result)
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = compat.pickle_load(reader.string())
arr = col.to_numpy_for_pandas()
ex_result = pd.Series(arr).mean()
assert result == ex_result
def test_finalize(self):
klass = self._get_mean_uda()
col = self.col_fragments[0]
result = self._update(klass, [col])
task, mm = self._make_finalize_task(result)
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = compat.pickle_load(reader.string())
arr = col.to_numpy_for_pandas()
ex_result = pd.Series(arr).mean()
assert result == ex_result
def __init__(self, shmem):
AggregationTask.__init__(self, shmem)
reader = wire.PackedMessageReader(shmem)
self.state = compat.pickle_load(reader.string())