def test_gets_event(self):
queue = Queue(capacity=10)
event = QueueEvent('test')
queue.puts((event, 1, 2, 3, event))
result = queue.gets(count=5, timeout=TIMEOUT)
self.assertEqual(result, (event, ))
result = queue.gets(count=5, timeout=TIMEOUT)
self.assertEqual(result, (1, 2, 3, event))
task = PeriodicTask(TIMEOUT / 10, lambda: queue.put(1))
task.start()
try:
result = queue.gets(count=5, timeout=TIMEOUT)
self.assertEqual(result, (1, 1, 1, 1, 1))
finally:
task.cancel()
task.join()
def test_flush_timeout(self):
queue = Queue(capacity=10)
self.assertEqual(queue.flush(timeout=TIMEOUT), 0)
queue.put('test', timeout=TIMEOUT)
self.assertIsNone(queue.flush(timeout=TIMEOUT))
queue.puts(range(100), timeout=TIMEOUT)
self.assertIsNone(queue.flush(timeout=TIMEOUT))
def proc():
for item in queue.gets(count=1, timeout=TIMEOUT):
if isinstance(item, QueueEvent):
item.set()
task = PeriodicTask(TIMEOUT / 10, proc)
task.start()
try:
self.assertIsNotNone(queue.flush())
finally:
task.cancel()
task.join()
class LocalFileStorage(object):
def __init__(
self,
path,
max_size=100 * 1024 * 1024, # 100MB
maintenance_period=60, # 1 minute
retention_period=7 * 24 * 60 * 60, # 7 days
write_timeout=60, # 1 minute
):
self.path = os.path.abspath(path)
self.max_size = max_size
self.maintenance_period = maintenance_period
self.retention_period = retention_period
self.write_timeout = write_timeout
self._maintenance_routine(silent=False)
self._maintenance_task = PeriodicTask(
interval=self.maintenance_period,
function=self._maintenance_routine,
kwargs={'silent': True},
)
self._maintenance_task.daemon = True
self._maintenance_task.start()
def close(self):
self._maintenance_task.cancel()
self._maintenance_task.join()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.close()
def _maintenance_routine(self, silent=False):
try:
if not os.path.isdir(self.path):
os.makedirs(self.path)
except Exception:
if not silent:
raise
try:
for blob in self.gets():
pass
except Exception:
if not silent:
raise
def gets(self):
now = _now()
lease_deadline = _fmt(now)
retention_deadline = _fmt(now - _seconds(self.retention_period))
timeout_deadline = _fmt(now - _seconds(self.write_timeout))
for name in sorted(os.listdir(self.path)):
path = os.path.join(self.path, name)
if not os.path.isfile(path):
continue # skip if not a file
if path.endswith('.tmp'):
if name < timeout_deadline:
try:
os.remove(path) # TODO: log data loss
except Exception:
pass # keep silent
if path.endswith('.lock'):
if path[path.rindex('@') + 1:-5] > lease_deadline:
continue # under lease
new_path = path[:path.rindex('@')]
try:
os.rename(path, new_path)
except Exception:
continue # keep silent
path = new_path
if path.endswith('.blob'):
if name < retention_deadline:
try:
os.remove(path) # TODO: log data loss
except Exception:
pass # keep silent
else:
yield LocalFileBlob(path)
def get(self):
cursor = self.gets()
try:
return next(cursor)
except StopIteration:
pass
return None
def put(self, data, lease_period=0, silent=False):
blob = LocalFileBlob(
os.path.join(
self.path,
'{}-{}.blob'.format(
_fmt(_now()),
'{:08x}'.format(
random.getrandbits(32)), # thread-safe random
),
))
return blob.put(data, lease_period=lease_period, silent=silent)
class LocalFileStorage(object):
def __init__(
self,
path,
max_size=50 * 1024 * 1024, # 50MiB
maintenance_period=60, # 1 minute
retention_period=7 * 24 * 60 * 60, # 7 days
write_timeout=60, # 1 minute
):
self.path = os.path.abspath(path)
self.max_size = max_size
self.maintenance_period = maintenance_period
self.retention_period = retention_period
self.write_timeout = write_timeout
self._maintenance_routine(silent=False)
self._maintenance_task = PeriodicTask(
interval=self.maintenance_period,
function=self._maintenance_routine,
kwargs={'silent': True},
)
self._maintenance_task.daemon = True
self._maintenance_task.start()
def close(self):
self._maintenance_task.cancel()
self._maintenance_task.join()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.close()
def _maintenance_routine(self, silent=False):
try:
if not os.path.isdir(self.path):
os.makedirs(self.path)
except Exception:
if not silent:
raise
try:
for blob in self.gets():
pass
except Exception:
if not silent:
raise
def gets(self):
now = _now()
lease_deadline = _fmt(now)
retention_deadline = _fmt(now - _seconds(self.retention_period))
timeout_deadline = _fmt(now - _seconds(self.write_timeout))
for name in sorted(os.listdir(self.path)):
path = os.path.join(self.path, name)
if not os.path.isfile(path):
continue # skip if not a file
if path.endswith('.tmp'):
if name < timeout_deadline:
try:
os.remove(path) # TODO: log data loss
except Exception:
pass # keep silent
if path.endswith('.lock'):
if path[path.rindex('@') + 1:-5] > lease_deadline:
continue # under lease
new_path = path[:path.rindex('@')]
try:
os.rename(path, new_path)
except Exception:
continue # keep silent
path = new_path
if path.endswith('.blob'):
if name < retention_deadline:
try:
os.remove(path) # TODO: log data loss
except Exception:
pass # keep silent
else:
yield LocalFileBlob(path)
def get(self):
cursor = self.gets()
try:
return next(cursor)
except StopIteration:
pass
return None
def put(self, data, lease_period=0, silent=False):
if not self._check_storage_size():
return None
blob = LocalFileBlob(
os.path.join(
self.path,
'{}-{}.blob'.format(
_fmt(_now()),
'{:08x}'.format(
random.getrandbits(32)), # thread-safe random
),
))
return blob.put(data, lease_period=lease_period, silent=silent)
def _check_storage_size(self):
size = 0
for dirpath, dirnames, filenames in os.walk(self.path):
for f in filenames:
fp = os.path.join(dirpath, f)
# skip if it is symbolic link
if not os.path.islink(fp):
try:
size += os.path.getsize(fp)
except OSError:
logger.error(
"Path %s does not exist or is inaccessible.", fp)
continue
if size >= self.max_size:
logger.warning(
"Persistent storage max capacity has been "
"reached. Currently at %fKB. Telemetry will be "
"lost. Please consider increasing the value of "
"'storage_max_size' in exporter config.",
format(size / 1024))
return False
return True