def run_test():
counter = Counter()
ceiling = 1000
master = IPCLock(is_slave=0)
slave = IPCLock(master.semaphore_id)
def ping():
while True:
with slave:
counter.inc()
if counter.total > ceiling:
break
def pong():
while True:
with master:
counter.inc()
if counter.total > ceiling:
break
t1 = threading.Thread(target=pong)
t1.start()
t2 = threading.Thread(target=ping)
t2.start()
t1.join()
t2.join()
def setUp(self):
self.paths_to_delete = []
# make size small so tracebacks get truncated
path, size, offset = 'task_%s' % guid(), 36, 0
self.paths_to_delete.append(path)
self.lock = IPCLock(is_slave=0)
self.task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)
self.mm = SharedMmap(path, size, create=True)
wire.PackedMessageWriter(self.mm).string('ping')
def setUp(self):
self.paths_to_delete = []
self.col_fragments = [double_ex(1000) for _ in range(10)]
self.lock = IPCLock(is_slave=0)
class TestAggregateTasks(unittest.TestCase):
def _get_mean_uda(self):
# Dynamically generate the class instance. Use pandas so nulls are
# excluded
class Mean(object):
def __init__(self):
self.total = 0
self.count = 0
def update(self, values):
values = pd.Series(values)
self.total += values.sum()
self.count += values.count()
def merge(self, other):
self.total += other.total
self.count += other.count
return self
def finalize(self):
return self.total / float(self.count)
return Mean
def setUp(self):
self.paths_to_delete = []
self.col_fragments = [double_ex(1000) for _ in range(10)]
self.lock = IPCLock(is_slave=0)
def test_update(self):
klass = self._get_mean_uda()
col = self.col_fragments[0]
task, mm = self._make_update_task(klass, [col])
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = pickle_load(reader.string())
ex_total = pd.Series(col.to_numpy_for_pandas()).sum()
assert result.total == ex_total
# Test with prior state
col = self.col_fragments[1]
task, mm = self._make_update_task(klass, [col], prior_state=result)
# Executor's turn again
self.lock.release()
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = pickle_load(reader.string())
ex_total += pd.Series(col.to_numpy_for_pandas()).sum()
# pandas will yield 0 on None input strangely
assert ex_total != 0
assert result.total == ex_total
def test_merge(self):
klass = self._get_mean_uda()
lcol = self.col_fragments[0]
rcol = self.col_fragments[1]
left = self._update(klass, [lcol])
right = self._update(klass, [rcol])
task, mm = self._make_merge_task(left, right)
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = pickle_load(reader.string())
larr = lcol.to_numpy_for_pandas()
rarr = rcol.to_numpy_for_pandas()
assert larr is not None
ex_total = (pd.Series(larr).sum() + pd.Series(rarr).sum())
assert result.total == ex_total
def test_finalize(self):
klass = self._get_mean_uda()
col = self.col_fragments[0]
result = self._update(klass, [col])
task, mm = self._make_finalize_task(result)
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = pickle_load(reader.string())
arr = col.to_numpy_for_pandas()
ex_result = pd.Series(arr).mean()
assert result == ex_result
def _update(self, klass, args):
task, mm = self._make_update_task(klass, args)
_execute_task(task, self.lock)
self.lock.release()
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
return reader.string()
def _make_update_task(self, uda_class, cols, prior_state=None):
# Overall layout here:
# - task name
# - serialized agg class
# - prior state flag 1/0
# - (optional) serialized prior state
# - serialized table fragment
payload = BytesIO()
msg_writer = wire.PackedMessageWriter(payload)
msg_writer.string('agg-update')
msg_writer.string(pickle_dump(uda_class))
if prior_state is not None:
msg_writer.uint8(1)
msg_writer.string(pickle_dump(prior_state))
else:
msg_writer.uint8(0)
writer = IbisTableWriter(cols)
# Create memory map of the appropriate size
path = 'task_%s' % guid()
size = writer.total_size() + payload.tell()
offset = 0
mm = SharedMmap(path, size, create=True)
self.paths_to_delete.append(path)
mm.write(payload.getvalue())
writer.write(mm)
task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)
return task, mm
def _make_merge_task(self, left_pickled, right_pickled):
payload = BytesIO()
msg_writer = wire.PackedMessageWriter(payload)
msg_writer.string('agg-merge')
msg_writer.string(left_pickled)
msg_writer.string(right_pickled)
# Create memory map of the appropriate size
path = 'task_%s' % guid()
size = payload.tell()
offset = 0
mm = SharedMmap(path, size, create=True)
self.paths_to_delete.append(path)
mm.write(payload.getvalue())
task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)
return task, mm
def _make_finalize_task(self, pickled):
payload = BytesIO()
msg_writer = wire.PackedMessageWriter(payload)
msg_writer.string('agg-finalize')
msg_writer.string(pickled)
# Create memory map of the appropriate size
path = 'task_%s' % guid()
size = payload.tell()
offset = 0
mm = SharedMmap(path, size, create=True)
self.paths_to_delete.append(path)
mm.write(payload.getvalue())
task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)
return task, mm
def tearDown(self):
for path in self.paths_to_delete:
try:
os.remove(path)
except os.error:
pass
def setUp(self):
self.paths_to_delete = []
self.col_fragments = [double_ex(1000) for _ in range(10)]
self.lock = IPCLock(is_slave=0)
class TestAggregateTasks(unittest.TestCase):
def _get_mean_uda(self):
# Dynamically generate the class instance. Use pandas so nulls are
# excluded
class Mean(object):
def __init__(self):
self.total = 0
self.count = 0
def update(self, values):
values = pd.Series(values)
self.total += values.sum()
self.count += values.count()
def merge(self, other):
self.total += other.total
self.count += other.count
return self
def finalize(self):
return self.total / float(self.count)
return Mean
def setUp(self):
self.paths_to_delete = []
self.col_fragments = [double_ex(1000) for _ in range(10)]
self.lock = IPCLock(is_slave=0)
def test_update(self):
klass = self._get_mean_uda()
col = self.col_fragments[0]
task, mm = self._make_update_task(klass, [col])
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = compat.pickle_load(reader.string())
ex_total = pd.Series(col.to_numpy_for_pandas()).sum()
assert result.total == ex_total
# Test with prior state
col = self.col_fragments[1]
task, mm = self._make_update_task(klass, [col], prior_state=result)
# Executor's turn again
self.lock.release()
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = compat.pickle_load(reader.string())
ex_total += pd.Series(col.to_numpy_for_pandas()).sum()
# pandas will yield 0 on None input strangely
assert ex_total != 0
assert result.total == ex_total
def test_merge(self):
klass = self._get_mean_uda()
lcol = self.col_fragments[0]
rcol = self.col_fragments[1]
left = self._update(klass, [lcol])
right = self._update(klass, [rcol])
task, mm = self._make_merge_task(left, right)
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = compat.pickle_load(reader.string())
larr = lcol.to_numpy_for_pandas()
rarr = rcol.to_numpy_for_pandas()
assert larr is not None
ex_total = (pd.Series(larr).sum() + pd.Series(rarr).sum())
assert result.total == ex_total
def test_finalize(self):
klass = self._get_mean_uda()
col = self.col_fragments[0]
result = self._update(klass, [col])
task, mm = self._make_finalize_task(result)
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = compat.pickle_load(reader.string())
arr = col.to_numpy_for_pandas()
ex_result = pd.Series(arr).mean()
assert result == ex_result
def _update(self, klass, args):
task, mm = self._make_update_task(klass, args)
_execute_task(task, self.lock)
self.lock.release()
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
return reader.string()
def _make_update_task(self, uda_class, cols, prior_state=None):
# Overall layout here:
# - task name
# - serialized agg class
# - prior state flag 1/0
# - (optional) serialized prior state
# - serialized table fragment
payload = BytesIO()
msg_writer = wire.PackedMessageWriter(payload)
msg_writer.string('agg-update')
msg_writer.string(compat.pickle_dump(uda_class))
if prior_state is not None:
msg_writer.uint8(1)
msg_writer.string(compat.pickle_dump(prior_state))
else:
msg_writer.uint8(0)
writer = IbisTableWriter(cols)
# Create memory map of the appropriate size
path = 'task_%s' % guid()
size = writer.total_size() + payload.tell()
offset = 0
mm = SharedMmap(path, size, create=True)
self.paths_to_delete.append(path)
mm.write(payload.getvalue())
writer.write(mm)
task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)
return task, mm
def _make_merge_task(self, left_pickled, right_pickled):
payload = BytesIO()
msg_writer = wire.PackedMessageWriter(payload)
msg_writer.string('agg-merge')
msg_writer.string(left_pickled)
msg_writer.string(right_pickled)
# Create memory map of the appropriate size
path = 'task_%s' % guid()
size = payload.tell()
offset = 0
mm = SharedMmap(path, size, create=True)
self.paths_to_delete.append(path)
mm.write(payload.getvalue())
task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)
return task, mm
def _make_finalize_task(self, pickled):
payload = BytesIO()
msg_writer = wire.PackedMessageWriter(payload)
msg_writer.string('agg-finalize')
msg_writer.string(pickled)
# Create memory map of the appropriate size
path = 'task_%s' % guid()
size = payload.tell()
offset = 0
mm = SharedMmap(path, size, create=True)
self.paths_to_delete.append(path)
mm.write(payload.getvalue())
task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)
return task, mm
def tearDown(self):
for path in self.paths_to_delete:
try:
os.remove(path)
except os.error:
pass