Пример #1
0
    def test_gets_event(self):
        queue = Queue(capacity=10)
        event = QueueEvent('test')
        queue.puts((event, 1, 2, 3, event))
        result = queue.gets(count=5, timeout=TIMEOUT)
        self.assertEqual(result, (event, ))
        result = queue.gets(count=5, timeout=TIMEOUT)
        self.assertEqual(result, (1, 2, 3, event))

        task = PeriodicTask(TIMEOUT / 10, lambda: queue.put(1))
        task.start()
        try:
            result = queue.gets(count=5, timeout=TIMEOUT)
            self.assertEqual(result, (1, 1, 1, 1, 1))
        finally:
            task.cancel()
            task.join()
Пример #2
0
    def test_flush_timeout(self):
        queue = Queue(capacity=10)
        self.assertEqual(queue.flush(timeout=TIMEOUT), 0)
        queue.put('test', timeout=TIMEOUT)
        self.assertIsNone(queue.flush(timeout=TIMEOUT))
        queue.puts(range(100), timeout=TIMEOUT)
        self.assertIsNone(queue.flush(timeout=TIMEOUT))

        def proc():
            for item in queue.gets(count=1, timeout=TIMEOUT):
                if isinstance(item, QueueEvent):
                    item.set()

        task = PeriodicTask(TIMEOUT / 10, proc)
        task.start()
        try:
            self.assertIsNotNone(queue.flush())
        finally:
            task.cancel()
            task.join()
Пример #3
0
class LocalFileStorage(object):
    def __init__(
            self,
            path,
            max_size=100 * 1024 * 1024,  # 100MB
            maintenance_period=60,  # 1 minute
            retention_period=7 * 24 * 60 * 60,  # 7 days
            write_timeout=60,  # 1 minute
    ):
        self.path = os.path.abspath(path)
        self.max_size = max_size
        self.maintenance_period = maintenance_period
        self.retention_period = retention_period
        self.write_timeout = write_timeout
        self._maintenance_routine(silent=False)
        self._maintenance_task = PeriodicTask(
            interval=self.maintenance_period,
            function=self._maintenance_routine,
            kwargs={'silent': True},
        )
        self._maintenance_task.daemon = True
        self._maintenance_task.start()

    def close(self):
        self._maintenance_task.cancel()
        self._maintenance_task.join()

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.close()

    def _maintenance_routine(self, silent=False):
        try:
            if not os.path.isdir(self.path):
                os.makedirs(self.path)
        except Exception:
            if not silent:
                raise
        try:
            for blob in self.gets():
                pass
        except Exception:
            if not silent:
                raise

    def gets(self):
        now = _now()
        lease_deadline = _fmt(now)
        retention_deadline = _fmt(now - _seconds(self.retention_period))
        timeout_deadline = _fmt(now - _seconds(self.write_timeout))
        for name in sorted(os.listdir(self.path)):
            path = os.path.join(self.path, name)
            if not os.path.isfile(path):
                continue  # skip if not a file
            if path.endswith('.tmp'):
                if name < timeout_deadline:
                    try:
                        os.remove(path)  # TODO: log data loss
                    except Exception:
                        pass  # keep silent
            if path.endswith('.lock'):
                if path[path.rindex('@') + 1:-5] > lease_deadline:
                    continue  # under lease
                new_path = path[:path.rindex('@')]
                try:
                    os.rename(path, new_path)
                except Exception:
                    continue  # keep silent
                path = new_path
            if path.endswith('.blob'):
                if name < retention_deadline:
                    try:
                        os.remove(path)  # TODO: log data loss
                    except Exception:
                        pass  # keep silent
                else:
                    yield LocalFileBlob(path)

    def get(self):
        cursor = self.gets()
        try:
            return next(cursor)
        except StopIteration:
            pass
        return None

    def put(self, data, lease_period=0, silent=False):
        blob = LocalFileBlob(
            os.path.join(
                self.path,
                '{}-{}.blob'.format(
                    _fmt(_now()),
                    '{:08x}'.format(
                        random.getrandbits(32)),  # thread-safe random
                ),
            ))
        return blob.put(data, lease_period=lease_period, silent=silent)
Пример #4
0
class LocalFileStorage(object):
    def __init__(
            self,
            path,
            max_size=50 * 1024 * 1024,  # 50MiB
            maintenance_period=60,  # 1 minute
            retention_period=7 * 24 * 60 * 60,  # 7 days
            write_timeout=60,  # 1 minute
    ):
        self.path = os.path.abspath(path)
        self.max_size = max_size
        self.maintenance_period = maintenance_period
        self.retention_period = retention_period
        self.write_timeout = write_timeout
        self._maintenance_routine(silent=False)
        self._maintenance_task = PeriodicTask(
            interval=self.maintenance_period,
            function=self._maintenance_routine,
            kwargs={'silent': True},
        )
        self._maintenance_task.daemon = True
        self._maintenance_task.start()

    def close(self):
        self._maintenance_task.cancel()
        self._maintenance_task.join()

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.close()

    def _maintenance_routine(self, silent=False):
        try:
            if not os.path.isdir(self.path):
                os.makedirs(self.path)
        except Exception:
            if not silent:
                raise
        try:
            for blob in self.gets():
                pass
        except Exception:
            if not silent:
                raise

    def gets(self):
        now = _now()
        lease_deadline = _fmt(now)
        retention_deadline = _fmt(now - _seconds(self.retention_period))
        timeout_deadline = _fmt(now - _seconds(self.write_timeout))
        for name in sorted(os.listdir(self.path)):
            path = os.path.join(self.path, name)
            if not os.path.isfile(path):
                continue  # skip if not a file
            if path.endswith('.tmp'):
                if name < timeout_deadline:
                    try:
                        os.remove(path)  # TODO: log data loss
                    except Exception:
                        pass  # keep silent
            if path.endswith('.lock'):
                if path[path.rindex('@') + 1:-5] > lease_deadline:
                    continue  # under lease
                new_path = path[:path.rindex('@')]
                try:
                    os.rename(path, new_path)
                except Exception:
                    continue  # keep silent
                path = new_path
            if path.endswith('.blob'):
                if name < retention_deadline:
                    try:
                        os.remove(path)  # TODO: log data loss
                    except Exception:
                        pass  # keep silent
                else:
                    yield LocalFileBlob(path)

    def get(self):
        cursor = self.gets()
        try:
            return next(cursor)
        except StopIteration:
            pass
        return None

    def put(self, data, lease_period=0, silent=False):
        if not self._check_storage_size():
            return None
        blob = LocalFileBlob(
            os.path.join(
                self.path,
                '{}-{}.blob'.format(
                    _fmt(_now()),
                    '{:08x}'.format(
                        random.getrandbits(32)),  # thread-safe random
                ),
            ))
        return blob.put(data, lease_period=lease_period, silent=silent)

    def _check_storage_size(self):
        size = 0
        for dirpath, dirnames, filenames in os.walk(self.path):
            for f in filenames:
                fp = os.path.join(dirpath, f)
                # skip if it is symbolic link
                if not os.path.islink(fp):
                    try:
                        size += os.path.getsize(fp)
                    except OSError:
                        logger.error(
                            "Path %s does not exist or is inaccessible.", fp)
                        continue
                    if size >= self.max_size:
                        logger.warning(
                            "Persistent storage max capacity has been "
                            "reached. Currently at %fKB. Telemetry will be "
                            "lost. Please consider increasing the value of "
                            "'storage_max_size' in exporter config.",
                            format(size / 1024))
                        return False
        return True