def _make_update_task(self, uda_class, cols, prior_state=None): # Overall layout here: # - task name # - serialized agg class # - prior state flag 1/0 # - (optional) serialized prior state # - serialized table fragment payload = BytesIO() msg_writer = wire.PackedMessageWriter(payload) msg_writer.string('agg-update') msg_writer.string(compat.pickle_dump(uda_class)) if prior_state is not None: msg_writer.uint8(1) msg_writer.string(compat.pickle_dump(prior_state)) else: msg_writer.uint8(0) writer = IbisTableWriter(cols) # Create memory map of the appropriate size path = 'task_%s' % guid() size = writer.total_size() + payload.tell() offset = 0 mm = SharedMmap(path, size, create=True) self.paths_to_delete.append(path) mm.write(payload.getvalue()) writer.write(mm) task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size) return task, mm
class TestPingPongTask(unittest.TestCase): def setUp(self): self.paths_to_delete = [] # make size small so tracebacks get truncated path, size, offset = 'task_%s' % guid(), 36, 0 self.paths_to_delete.append(path) self.lock = IPCLock(is_slave=0) self.task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size) self.mm = SharedMmap(path, size, create=True) wire.PackedMessageWriter(self.mm).string('ping') def tearDown(self): for path in self.paths_to_delete: try: os.remove(path) except os.error: pass def test_execute_task(self): _execute_task(self.task, self.lock) self.mm.seek(0) reader = wire.PackedMessageReader(self.mm) assert reader.uint8() assert reader.string() == 'pong'
def _make_update_task(self, uda_class, cols, prior_state=None): # Overall layout here: # - task name # - serialized agg class # - prior state flag 1/0 # - (optional) serialized prior state # - serialized table fragment payload = BytesIO() msg_writer = wire.PackedMessageWriter(payload) msg_writer.string('agg-update') msg_writer.string(pickle_dump(uda_class)) if prior_state is not None: msg_writer.uint8(1) msg_writer.string(pickle_dump(prior_state)) else: msg_writer.uint8(0) writer = IbisTableWriter(cols) # Create memory map of the appropriate size path = 'task_%s' % guid() size = writer.total_size() + payload.tell() offset = 0 mm = SharedMmap(path, size, create=True) self.paths_to_delete.append(path) mm.write(payload.getvalue()) writer.write(mm) task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size) return task, mm
def test_create_file(self): size = 1024 path = guid() try: mm = SharedMmap(path, size, create=True) mm.close() self.assertTrue(os.path.exists(path)) self.assertEqual(os.stat(path).st_size, size) finally: _nuke(path)
def setUp(self): self.paths_to_delete = [] # make size small so tracebacks get truncated path, size, offset = 'task_%s' % guid(), 36, 0 self.paths_to_delete.append(path) self.lock = IPCLock(is_slave=0) self.task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size) self.mm = SharedMmap(path, size, create=True) wire.PackedMessageWriter(self.mm).string('ping')
def test_file_interface(self): path = guid() self.to_nuke.append(path) data = guid() mm = SharedMmap(path, len(data), create=True) assert mm.tell() == 0 mm.write(data) assert mm.tell() == len(data) mm.seek(0) assert mm.tell() == 0 result = mm.read(16) assert len(result) == 16 assert result == data[:16] assert mm.tell() == 16
def test_close_file(self): path = guid() self.to_nuke.append(path) data = guid() mm = SharedMmap(path, len(data), create=True) assert mm.closed is False mm.close() assert mm.closed is True # idempotent mm.close() assert mm.closed is True self.assertRaises(IOError, mm.read, 4) self.assertRaises(IOError, mm.write, 'bazqux') self.assertRaises(IOError, mm.seek, 0) self.assertRaises(IOError, mm.flush)
def _make_finalize_task(self, pickled): payload = BytesIO() msg_writer = wire.PackedMessageWriter(payload) msg_writer.string('agg-finalize') msg_writer.string(pickled) # Create memory map of the appropriate size path = 'task_%s' % guid() size = payload.tell() offset = 0 mm = SharedMmap(path, size, create=True) self.paths_to_delete.append(path) mm.write(payload.getvalue()) task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size) return task, mm
def test_multiple_mmaps(self): path = guid() path2 = guid() data = guid() self.to_nuke.extend([path, path2]) mm1 = SharedMmap(path, len(data), create=True) mm1.write(data) mm2 = SharedMmap(path, len(data)) result = mm2.read() self.assertEqual(result, data) # Open both maps first, see if data synchronizes mm1 = SharedMmap(path2, len(data), create=True) mm2 = SharedMmap(path2, len(data)) mm1.write(data) result = mm2.read() self.assertEqual(result, data)