def __init__(
self,
path,
max_size=50 * 1024 * 1024, # 50MiB
maintenance_period=60, # 1 minute
retention_period=7 * 24 * 60 * 60, # 7 days
write_timeout=60, # 1 minute
):
self.path = os.path.abspath(path)
self.max_size = max_size
self.maintenance_period = maintenance_period
self.retention_period = retention_period
self.write_timeout = write_timeout
self._maintenance_routine()
self._maintenance_task = PeriodicTask(
interval=self.maintenance_period,
function=self._maintenance_routine,
)
self._maintenance_task.daemon = True
self._maintenance_task.start()
class LocalFileStorage:
def __init__(
self,
path,
max_size=50 * 1024 * 1024, # 50MiB
maintenance_period=60, # 1 minute
retention_period=7 * 24 * 60 * 60, # 7 days
write_timeout=60, # 1 minute
):
self.path = os.path.abspath(path)
self.max_size = max_size
self.maintenance_period = maintenance_period
self.retention_period = retention_period
self.write_timeout = write_timeout
self._maintenance_routine(silent=False)
self._maintenance_task = PeriodicTask(
interval=self.maintenance_period,
function=self._maintenance_routine,
kwargs={"silent": True},
)
self._maintenance_task.daemon = True
self._maintenance_task.start()
def close(self):
self._maintenance_task.cancel()
self._maintenance_task.join()
def __enter__(self):
return self
# pylint: disable=redefined-builtin
def __exit__(self, type, value, traceback):
self.close()
def _maintenance_routine(self, silent=False):
try:
if not os.path.isdir(self.path):
os.makedirs(self.path)
except Exception:
if not silent:
raise
try:
# pylint: disable=unused-variable
for blob in self.gets():
pass
except Exception:
if not silent:
raise
def gets(self):
now = _now()
lease_deadline = _fmt(now)
retention_deadline = _fmt(now - _seconds(self.retention_period))
timeout_deadline = _fmt(now - _seconds(self.write_timeout))
for name in sorted(os.listdir(self.path)):
path = os.path.join(self.path, name)
if not os.path.isfile(path):
continue # skip if not a file
if path.endswith(".tmp"):
if name < timeout_deadline:
try:
os.remove(path) # TODO: log data loss
except Exception:
pass # keep silent
if path.endswith(".lock"):
if path[path.rindex("@") + 1 : -5] > lease_deadline:
continue # under lease
new_path = path[: path.rindex("@")]
try:
os.rename(path, new_path)
except Exception:
continue # keep silent
path = new_path
if path.endswith(".blob"):
if name < retention_deadline:
try:
os.remove(path) # TODO: log data loss
except Exception:
pass # keep silent
else:
yield LocalFileBlob(path)
def get(self):
cursor = self.gets()
try:
return next(cursor)
except StopIteration:
pass
return None
def put(self, data, lease_period=0, silent=False):
if not self._check_storage_size():
return None
blob = LocalFileBlob(
os.path.join(
self.path,
"{}-{}.blob".format(
_fmt(_now()),
"{:08x}".format(
random.getrandbits(32)
), # thread-safe random
),
)
)
return blob.put(data, lease_period=lease_period, silent=silent)
def _check_storage_size(self):
size = 0
# pylint: disable=unused-variable
for dirpath, dirnames, filenames in os.walk(self.path):
for filename in filenames:
path = os.path.join(dirpath, filename)
# skip if it is symbolic link
if not os.path.islink(path):
try:
size += os.path.getsize(path)
except OSError:
logger.error(
"Path %s does not exist or is " "inaccessible.",
path,
)
continue
if size >= self.max_size:
logger.warning(
"Persistent storage max capacity has been "
"reached. Currently at %fKB. Telemetry will be "
"lost. Please consider increasing the value of "
"'storage_max_size' in exporter config.",
format(size / 1024),
)
return False
return True
class LocalFileStorage(object):
def __init__(
self,
path,
max_size=100 * 1024 * 1024, # 100MB
maintenance_period=60, # 1 minute
retention_period=7 * 24 * 60 * 60, # 7 days
write_timeout=60, # 1 minute
):
self.path = os.path.abspath(path)
self.max_size = max_size
self.maintenance_period = maintenance_period
self.retention_period = retention_period
self.write_timeout = write_timeout
self._maintenance_routine(silent=False)
self._maintenance_task = PeriodicTask(
interval=self.maintenance_period,
function=self._maintenance_routine,
kwargs={"silent": True},
)
self._maintenance_task.daemon = True
self._maintenance_task.start()
def close(self):
self._maintenance_task.cancel()
self._maintenance_task.join()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.close()
def _maintenance_routine(self, silent=False):
try:
if not os.path.isdir(self.path):
os.makedirs(self.path)
except Exception:
if not silent:
raise
try:
for blob in self.gets():
pass
except Exception:
if not silent:
raise
def gets(self):
now = _now()
lease_deadline = _fmt(now)
retention_deadline = _fmt(now - _seconds(self.retention_period))
timeout_deadline = _fmt(now - _seconds(self.write_timeout))
for name in sorted(os.listdir(self.path)):
path = os.path.join(self.path, name)
if not os.path.isfile(path):
continue # skip if not a file
if path.endswith(".tmp"):
if name < timeout_deadline:
try:
os.remove(path) # TODO: log data loss
except Exception:
pass # keep silent
if path.endswith(".lock"):
if path[path.rindex("@") + 1:-5] > lease_deadline:
continue # under lease
new_path = path[:path.rindex("@")]
try:
os.rename(path, new_path)
except Exception:
continue # keep silent
path = new_path
if path.endswith(".blob"):
if name < retention_deadline:
try:
os.remove(path) # TODO: log data loss
except Exception:
pass # keep silent
else:
yield LocalFileBlob(path)
def get(self):
cursor = self.gets()
try:
return next(cursor)
except StopIteration:
pass
return None
def put(self, data, lease_period=0, silent=False):
blob = LocalFileBlob(
os.path.join(
self.path,
"{}-{}.blob".format(
_fmt(_now()),
"{:08x}".format(
random.getrandbits(32)), # thread-safe random
),
))
return blob.put(data, lease_period=lease_period, silent=silent)