def _make_update_task(self, uda_class, cols, prior_state=None):
# Overall layout here:
# - task name
# - serialized agg class
# - prior state flag 1/0
# - (optional) serialized prior state
# - serialized table fragment
payload = BytesIO()
msg_writer = wire.PackedMessageWriter(payload)
msg_writer.string('agg-update')
msg_writer.string(compat.pickle_dump(uda_class))
if prior_state is not None:
msg_writer.uint8(1)
msg_writer.string(compat.pickle_dump(prior_state))
else:
msg_writer.uint8(0)
writer = IbisTableWriter(cols)
# Create memory map of the appropriate size
path = 'task_%s' % guid()
size = writer.total_size() + payload.tell()
offset = 0
mm = SharedMmap(path, size, create=True)
self.paths_to_delete.append(path)
mm.write(payload.getvalue())
writer.write(mm)
task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)
return task, mm
class TestPingPongTask(unittest.TestCase):
def setUp(self):
self.paths_to_delete = []
# make size small so tracebacks get truncated
path, size, offset = 'task_%s' % guid(), 36, 0
self.paths_to_delete.append(path)
self.lock = IPCLock(is_slave=0)
self.task = IbisTaskMessage(self.lock.semaphore_id, path,
offset, size)
self.mm = SharedMmap(path, size, create=True)
wire.PackedMessageWriter(self.mm).string('ping')
def tearDown(self):
for path in self.paths_to_delete:
try:
os.remove(path)
except os.error:
pass
def test_execute_task(self):
_execute_task(self.task, self.lock)
self.mm.seek(0)
reader = wire.PackedMessageReader(self.mm)
assert reader.uint8()
assert reader.string() == 'pong'
def _make_update_task(self, uda_class, cols, prior_state=None):
# Overall layout here:
# - task name
# - serialized agg class
# - prior state flag 1/0
# - (optional) serialized prior state
# - serialized table fragment
payload = BytesIO()
msg_writer = wire.PackedMessageWriter(payload)
msg_writer.string('agg-update')
msg_writer.string(pickle_dump(uda_class))
if prior_state is not None:
msg_writer.uint8(1)
msg_writer.string(pickle_dump(prior_state))
else:
msg_writer.uint8(0)
writer = IbisTableWriter(cols)
# Create memory map of the appropriate size
path = 'task_%s' % guid()
size = writer.total_size() + payload.tell()
offset = 0
mm = SharedMmap(path, size, create=True)
self.paths_to_delete.append(path)
mm.write(payload.getvalue())
writer.write(mm)
task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)
return task, mm
class TestPingPongTask(unittest.TestCase):
def setUp(self):
self.paths_to_delete = []
# make size small so tracebacks get truncated
path, size, offset = 'task_%s' % guid(), 36, 0
self.paths_to_delete.append(path)
self.lock = IPCLock(is_slave=0)
self.task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)
self.mm = SharedMmap(path, size, create=True)
wire.PackedMessageWriter(self.mm).string('ping')
def tearDown(self):
for path in self.paths_to_delete:
try:
os.remove(path)
except os.error:
pass
def test_execute_task(self):
_execute_task(self.task, self.lock)
self.mm.seek(0)
reader = wire.PackedMessageReader(self.mm)
assert reader.uint8()
assert reader.string() == 'pong'
def test_create_file(self):
size = 1024
path = guid()
try:
mm = SharedMmap(path, size, create=True)
mm.close()
self.assertTrue(os.path.exists(path))
self.assertEqual(os.stat(path).st_size, size)
finally:
_nuke(path)
def setUp(self):
self.paths_to_delete = []
# make size small so tracebacks get truncated
path, size, offset = 'task_%s' % guid(), 36, 0
self.paths_to_delete.append(path)
self.lock = IPCLock(is_slave=0)
self.task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)
self.mm = SharedMmap(path, size, create=True)
wire.PackedMessageWriter(self.mm).string('ping')
def test_create_file(self):
size = 1024
path = guid()
try:
mm = SharedMmap(path, size, create=True)
mm.close()
self.assertTrue(os.path.exists(path))
self.assertEqual(os.stat(path).st_size, size)
finally:
_nuke(path)
def test_file_interface(self):
path = guid()
self.to_nuke.append(path)
data = guid()
mm = SharedMmap(path, len(data), create=True)
assert mm.tell() == 0
mm.write(data)
assert mm.tell() == len(data)
mm.seek(0)
assert mm.tell() == 0
result = mm.read(16)
assert len(result) == 16
assert result == data[:16]
assert mm.tell() == 16
def test_close_file(self):
path = guid()
self.to_nuke.append(path)
data = guid()
mm = SharedMmap(path, len(data), create=True)
assert mm.closed is False
mm.close()
assert mm.closed is True
# idempotent
mm.close()
assert mm.closed is True
self.assertRaises(IOError, mm.read, 4)
self.assertRaises(IOError, mm.write, 'bazqux')
self.assertRaises(IOError, mm.seek, 0)
self.assertRaises(IOError, mm.flush)
def _make_finalize_task(self, pickled):
payload = BytesIO()
msg_writer = wire.PackedMessageWriter(payload)
msg_writer.string('agg-finalize')
msg_writer.string(pickled)
# Create memory map of the appropriate size
path = 'task_%s' % guid()
size = payload.tell()
offset = 0
mm = SharedMmap(path, size, create=True)
self.paths_to_delete.append(path)
mm.write(payload.getvalue())
task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)
return task, mm
def _make_finalize_task(self, pickled):
payload = BytesIO()
msg_writer = wire.PackedMessageWriter(payload)
msg_writer.string('agg-finalize')
msg_writer.string(pickled)
# Create memory map of the appropriate size
path = 'task_%s' % guid()
size = payload.tell()
offset = 0
mm = SharedMmap(path, size, create=True)
self.paths_to_delete.append(path)
mm.write(payload.getvalue())
task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)
return task, mm
def test_multiple_mmaps(self):
path = guid()
path2 = guid()
data = guid()
self.to_nuke.extend([path, path2])
mm1 = SharedMmap(path, len(data), create=True)
mm1.write(data)
mm2 = SharedMmap(path, len(data))
result = mm2.read()
self.assertEqual(result, data)
# Open both maps first, see if data synchronizes
mm1 = SharedMmap(path2, len(data), create=True)
mm2 = SharedMmap(path2, len(data))
mm1.write(data)
result = mm2.read()
self.assertEqual(result, data)
def setUp(self):
self.paths_to_delete = []
# make size small so tracebacks get truncated
path, size, offset = 'task_%s' % guid(), 36, 0
self.paths_to_delete.append(path)
self.lock = IPCLock(is_slave=0)
self.task = IbisTaskMessage(self.lock.semaphore_id, path,
offset, size)
self.mm = SharedMmap(path, size, create=True)
wire.PackedMessageWriter(self.mm).string('ping')
def test_close_file(self):
path = guid()
self.to_nuke.append(path)
data = guid()
mm = SharedMmap(path, len(data), create=True)
assert mm.closed is False
mm.close()
assert mm.closed is True
# idempotent
mm.close()
assert mm.closed is True
self.assertRaises(IOError, mm.read, 4)
self.assertRaises(IOError, mm.write, 'bazqux')
self.assertRaises(IOError, mm.seek, 0)
self.assertRaises(IOError, mm.flush)
def test_multiple_mmaps(self):
path = guid()
path2 = guid()
data = guid()
self.to_nuke.extend([path, path2])
mm1 = SharedMmap(path, len(data), create=True)
mm1.write(data)
mm2 = SharedMmap(path, len(data))
result = mm2.read()
self.assertEqual(result, data)
# Open both maps first, see if data synchronizes
mm1 = SharedMmap(path2, len(data), create=True)
mm2 = SharedMmap(path2, len(data))
mm1.write(data)
result = mm2.read()
self.assertEqual(result, data)
def test_file_interface(self):
path = guid()
self.to_nuke.append(path)
data = guid()
mm = SharedMmap(path, len(data), create=True)
assert mm.tell() == 0
mm.write(data)
assert mm.tell() == len(data)
mm.seek(0)
assert mm.tell() == 0
result = mm.read(16)
assert len(result) == 16
assert result == data[:16]
assert mm.tell() == 16
