Esempio n. 1
0
    def _make_update_task(self, uda_class, cols, prior_state=None):

        # Overall layout here:
        # - task name
        # - serialized agg class
        # - prior state flag 1/0
        # - (optional) serialized prior state
        # - serialized table fragment

        payload = BytesIO()
        msg_writer = wire.PackedMessageWriter(payload)
        msg_writer.string('agg-update')
        msg_writer.string(compat.pickle_dump(uda_class))

        if prior_state is not None:
            msg_writer.uint8(1)
            msg_writer.string(compat.pickle_dump(prior_state))
        else:
            msg_writer.uint8(0)

        writer = IbisTableWriter(cols)

        # Create memory map of the appropriate size
        path = 'task_%s' % guid()
        size = writer.total_size() + payload.tell()
        offset = 0
        mm = SharedMmap(path, size, create=True)
        self.paths_to_delete.append(path)

        mm.write(payload.getvalue())
        writer.write(mm)

        task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)

        return task, mm
Esempio n. 2
0
class TestPingPongTask(unittest.TestCase):

    def setUp(self):
        self.paths_to_delete = []

        # make size small so tracebacks get truncated
        path, size, offset = 'task_%s' % guid(), 36, 0
        self.paths_to_delete.append(path)

        self.lock = IPCLock(is_slave=0)
        self.task = IbisTaskMessage(self.lock.semaphore_id, path,
                                    offset, size)

        self.mm = SharedMmap(path, size, create=True)
        wire.PackedMessageWriter(self.mm).string('ping')

    def tearDown(self):
        for path in self.paths_to_delete:
            try:
                os.remove(path)
            except os.error:
                pass

    def test_execute_task(self):
        _execute_task(self.task, self.lock)

        self.mm.seek(0)
        reader = wire.PackedMessageReader(self.mm)
        assert reader.uint8()
        assert reader.string() == 'pong'
Esempio n. 3
0
    def _make_update_task(self, uda_class, cols, prior_state=None):

        # Overall layout here:
        # - task name
        # - serialized agg class
        # - prior state flag 1/0
        # - (optional) serialized prior state
        # - serialized table fragment

        payload = BytesIO()
        msg_writer = wire.PackedMessageWriter(payload)
        msg_writer.string('agg-update')
        msg_writer.string(pickle_dump(uda_class))

        if prior_state is not None:
            msg_writer.uint8(1)
            msg_writer.string(pickle_dump(prior_state))
        else:
            msg_writer.uint8(0)

        writer = IbisTableWriter(cols)

        # Create memory map of the appropriate size
        path = 'task_%s' % guid()
        size = writer.total_size() + payload.tell()
        offset = 0
        mm = SharedMmap(path, size, create=True)
        self.paths_to_delete.append(path)

        mm.write(payload.getvalue())
        writer.write(mm)

        task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)

        return task, mm
Esempio n. 4
0
class TestPingPongTask(unittest.TestCase):
    def setUp(self):
        self.paths_to_delete = []

        # make size small so tracebacks get truncated
        path, size, offset = 'task_%s' % guid(), 36, 0
        self.paths_to_delete.append(path)

        self.lock = IPCLock(is_slave=0)
        self.task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)

        self.mm = SharedMmap(path, size, create=True)
        wire.PackedMessageWriter(self.mm).string('ping')

    def tearDown(self):
        for path in self.paths_to_delete:
            try:
                os.remove(path)
            except os.error:
                pass

    def test_execute_task(self):
        _execute_task(self.task, self.lock)

        self.mm.seek(0)
        reader = wire.PackedMessageReader(self.mm)
        assert reader.uint8()
        assert reader.string() == 'pong'
Esempio n. 5
0
    def test_create_file(self):
        size = 1024

        path = guid()
        try:
            mm = SharedMmap(path, size, create=True)
            mm.close()

            self.assertTrue(os.path.exists(path))
            self.assertEqual(os.stat(path).st_size, size)
        finally:
            _nuke(path)
Esempio n. 6
0
    def setUp(self):
        self.paths_to_delete = []

        # make size small so tracebacks get truncated
        path, size, offset = 'task_%s' % guid(), 36, 0
        self.paths_to_delete.append(path)

        self.lock = IPCLock(is_slave=0)
        self.task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)

        self.mm = SharedMmap(path, size, create=True)
        wire.PackedMessageWriter(self.mm).string('ping')
Esempio n. 7
0
    def test_create_file(self):
        size = 1024

        path = guid()
        try:
            mm = SharedMmap(path, size, create=True)
            mm.close()

            self.assertTrue(os.path.exists(path))
            self.assertEqual(os.stat(path).st_size, size)
        finally:
            _nuke(path)
Esempio n. 8
0
    def test_file_interface(self):
        path = guid()
        self.to_nuke.append(path)
        data = guid()

        mm = SharedMmap(path, len(data), create=True)

        assert mm.tell() == 0
        mm.write(data)
        assert mm.tell() == len(data)

        mm.seek(0)
        assert mm.tell() == 0
        result = mm.read(16)
        assert len(result) == 16
        assert result == data[:16]
        assert mm.tell() == 16
Esempio n. 9
0
    def test_close_file(self):
        path = guid()
        self.to_nuke.append(path)
        data = guid()

        mm = SharedMmap(path, len(data), create=True)
        assert mm.closed is False
        mm.close()
        assert mm.closed is True

        # idempotent
        mm.close()
        assert mm.closed is True

        self.assertRaises(IOError, mm.read, 4)
        self.assertRaises(IOError, mm.write, 'bazqux')
        self.assertRaises(IOError, mm.seek, 0)
        self.assertRaises(IOError, mm.flush)
Esempio n. 10
0
    def _make_finalize_task(self, pickled):
        payload = BytesIO()
        msg_writer = wire.PackedMessageWriter(payload)
        msg_writer.string('agg-finalize')
        msg_writer.string(pickled)

        # Create memory map of the appropriate size
        path = 'task_%s' % guid()
        size = payload.tell()
        offset = 0
        mm = SharedMmap(path, size, create=True)
        self.paths_to_delete.append(path)

        mm.write(payload.getvalue())

        task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)

        return task, mm
Esempio n. 11
0
    def _make_finalize_task(self, pickled):
        payload = BytesIO()
        msg_writer = wire.PackedMessageWriter(payload)
        msg_writer.string('agg-finalize')
        msg_writer.string(pickled)

        # Create memory map of the appropriate size
        path = 'task_%s' % guid()
        size = payload.tell()
        offset = 0
        mm = SharedMmap(path, size, create=True)
        self.paths_to_delete.append(path)

        mm.write(payload.getvalue())

        task = IbisTaskMessage(self.lock.semaphore_id, path, offset, size)

        return task, mm
Esempio n. 12
0
    def test_multiple_mmaps(self):
        path = guid()
        path2 = guid()
        data = guid()
        self.to_nuke.extend([path, path2])

        mm1 = SharedMmap(path, len(data), create=True)
        mm1.write(data)

        mm2 = SharedMmap(path, len(data))
        result = mm2.read()
        self.assertEqual(result, data)

        # Open both maps first, see if data synchronizes
        mm1 = SharedMmap(path2, len(data), create=True)
        mm2 = SharedMmap(path2, len(data))

        mm1.write(data)
        result = mm2.read()
        self.assertEqual(result, data)
Esempio n. 13
0
    def setUp(self):
        self.paths_to_delete = []

        # make size small so tracebacks get truncated
        path, size, offset = 'task_%s' % guid(), 36, 0
        self.paths_to_delete.append(path)

        self.lock = IPCLock(is_slave=0)
        self.task = IbisTaskMessage(self.lock.semaphore_id, path,
                                    offset, size)

        self.mm = SharedMmap(path, size, create=True)
        wire.PackedMessageWriter(self.mm).string('ping')
Esempio n. 14
0
    def test_close_file(self):
        path = guid()
        self.to_nuke.append(path)
        data = guid()

        mm = SharedMmap(path, len(data), create=True)
        assert mm.closed is False
        mm.close()
        assert mm.closed is True

        # idempotent
        mm.close()
        assert mm.closed is True

        self.assertRaises(IOError, mm.read, 4)
        self.assertRaises(IOError, mm.write, 'bazqux')
        self.assertRaises(IOError, mm.seek, 0)
        self.assertRaises(IOError, mm.flush)
Esempio n. 15
0
    def test_multiple_mmaps(self):
        path = guid()
        path2 = guid()
        data = guid()
        self.to_nuke.extend([path, path2])

        mm1 = SharedMmap(path, len(data), create=True)
        mm1.write(data)

        mm2 = SharedMmap(path, len(data))
        result = mm2.read()
        self.assertEqual(result, data)

        # Open both maps first, see if data synchronizes
        mm1 = SharedMmap(path2, len(data), create=True)
        mm2 = SharedMmap(path2, len(data))

        mm1.write(data)
        result = mm2.read()
        self.assertEqual(result, data)
Esempio n. 16
0
    def test_file_interface(self):
        path = guid()
        self.to_nuke.append(path)
        data = guid()

        mm = SharedMmap(path, len(data), create=True)

        assert mm.tell() == 0
        mm.write(data)
        assert mm.tell() == len(data)

        mm.seek(0)
        assert mm.tell() == 0
        result = mm.read(16)
        assert len(result) == 16
        assert result == data[:16]
        assert mm.tell() == 16