def read(self, lock: bool = False) -> List[_VT]:
"""Parse the file and return the entries"""
path = self._path
if lock:
store.aquire_lock(path)
entries = []
try:
with path.open("rb") as f:
for entry in f.read().split(b"\0"):
if entry:
entries.append(
self._deserialize(
ast.literal_eval(entry.decode("utf-8"))))
except IOError as e:
if e.errno == errno.ENOENT:
pass
else:
raise
except Exception:
if lock:
store.release_lock(path)
raise
return entries
def rename_host_in_multisite(oldname, newname):
# State of Multisite ---------------------------------------
# Favorites of users and maybe other settings. We simply walk through
# all directories rather then through the user database. That way we
# are sure that also currently non-existant users are being found and
# also only users that really have a profile.
users_changed = 0
total_changed = 0
for userid in os.listdir(config.config_dir):
if userid[0] == '.':
continue
if not os.path.isdir(config.config_dir + "/" + userid):
continue
favpath = config.config_dir + "/" + userid + "/favorites.mk"
num_changed = 0
favorites = store.load_object_from_file(favpath, default=[], lock=True)
for nr, entry in enumerate(favorites):
if entry == oldname:
favorites[nr] = newname
num_changed += 1
elif entry.startswith(oldname + ";"):
favorites[nr] = newname + ";" + entry.split(";")[1]
num_changed += 1
if num_changed:
store.save_object_to_file(favpath, favorites)
users_changed += 1
total_changed += num_changed
store.release_lock(favpath)
if users_changed:
return ["favorites"] * total_changed
return []
def exclusive_lock():
path = cmk.utils.paths.default_config_dir + "/multisite.mk"
store.aquire_lock(path)
try:
yield
finally:
store.release_lock(path)
def store(
self,
*,
removed: Set[_ValueStoreKey],
updated: Mapping[_ValueStoreKey, Any],
) -> None:
"""Re-load and then store the changes of the item state to disk
Make sure the object is in sync with the file after writing.
"""
self._log_debug("Storing item states")
if not (removed or updated):
return
self._path.parent.mkdir(parents=True, exist_ok=True)
try:
store.aquire_lock(self._path)
self._load()
data = {
**{k: v
for k, v in self._data.items() if k not in removed},
**updated,
}
store.save_object_to_file(self._path, data, pretty=False)
self._mtime = self._path.stat().st_mtime
self._data = data
except Exception:
raise MKGeneralException(
f"Cannot write to {self._path}: {traceback.format_exc()}")
finally:
store.release_lock(self._path)
def start(self):
# type: () -> None
try:
store.aquire_lock(self._job_initializiation_lock)
self._start()
finally:
store.release_lock(self._job_initializiation_lock)
def update_cache(self, cache_id, ipa):
# type: (IPLookupCacheId, str) -> None
"""Updates the cache with a new / changed entry
When self.persist_on_update update is disabled, this simply updates the in-memory
cache without any persistence interaction. Otherwise:
The cache that was previously loaded into this IPLookupCache with load_persisted()
might be outdated compared to the current persisted lookup cache. Another process
might have updated the cache in the meantime.
The approach here is to load the currently persisted cache with a lock, then update
the current IPLookupCache with it, add the given new / changed entry and then write
out the resulting data structure.
This could really be solved in a better way, but may be sufficient for the moment.
The cache can only be cleaned up with the "Update DNS cache" option in WATO
or the "cmk --update-dns-cache" call that both call update_dns_cache().
"""
if not self.persist_on_update:
self[cache_id] = ipa
return
try:
self.update(_load_ip_lookup_cache(lock=True))
self[cache_id] = ipa
self.save_persisted()
finally:
store.release_lock(_cache_path())
def load(self, lock=False):
"""Parse the site specific changes file
The file format has been choosen to be able to append changes without
much cost. This is just a intermmediate format for 1.4.x. In 1.5 we will
reimplement WATO changes and this very specific file format will vanish."""
path = self._site_changes_path()
if lock:
store.aquire_lock(path)
changes = []
try:
for entry in open(path).read().split("\0"):
if entry:
changes.append(ast.literal_eval(entry))
except IOError as e:
if e.errno == errno.ENOENT:
pass
else:
raise
except:
if lock:
store.release_lock(path)
raise
return changes
def acquire(n):
assert not store.have_lock(path)
if debug:
print(f"{n}: Trying lock\n")
store.aquire_lock(path, blocking=True)
assert store.have_lock(path)
# We check to see if the other threads are actually waiting.
if not saw_someone_wait:
_wait_for_waiting_lock()
saw_someone_wait.append(1)
if debug:
print(f"{n}: Got lock\n")
acquired.append(1)
# This part is guarded by the lock, so we should never have more than one entry in here,
# even if multiple threads try to append at the same time
assert len(acquired) == 1
acquired.pop()
store.release_lock(path)
assert not store.have_lock(path)
if debug:
print(f"{n}: Released lock\n")
def load(self) -> None:
self._log_debug("Loading item states")
try:
store.aquire_lock(self._path)
self._load()
finally:
store.release_lock(self._path)
def test_release_lock(locked_file, path_type):
path = path_type(locked_file)
assert store.have_lock(path) is False
store.aquire_lock(path)
assert store.have_lock(path) is True
store.release_lock(path)
assert store.have_lock(path) is False
def test_non_blocking_locking_without_previous_lock(locked_file, path_type, t1):
assert t1.store != store
path = path_type(locked_file)
# Try to lock first
assert store.try_aquire_lock(path) is True
assert store.have_lock(path) is True
store.release_lock(path)
assert store.have_lock(path) is False
def acquire(_):
try:
store.aquire_lock(path, blocking=False)
acquired.append(1)
assert store.have_lock(path)
store.release_lock(path)
assert not store.have_lock(path)
except IOError:
assert not store.have_lock(path)
def test_release_lock(tmp_path):
locked_file = tmp_path / "locked_file"
locked_file.write_text(u"", encoding="utf-8")
path = str(locked_file)
assert store.have_lock(path) is False
store.aquire_lock(path)
assert store.have_lock(path) is True
store.release_lock(path)
assert store.have_lock(path) is False
def update_status(self, params):
if not self._jobstatus_path.parent.exists(): # pylint: disable=no-member
return
if params:
try:
status = store.load_data_from_file(str(self._jobstatus_path), {}, lock=True)
status.update(params)
store.save_mk_file(str(self._jobstatus_path), self._format_value(status))
finally:
store.release_lock(str(self._jobstatus_path))
def test_release_lock_already_closed(locked_file, path_type):
path = path_type(locked_file)
assert store.have_lock(path) is False
store.aquire_lock(path)
assert store.have_lock(path) is True
os.close(store._locks._get_lock(str(path))) # pylint:disable=no-value-for-parameter
store.release_lock(path)
assert store.have_lock(path) is False
def update_status(self, params: JobStatusSpec) -> None:
if not self._jobstatus_path.parent.exists():
return
if params:
try:
status = store.load_object_from_file(str(self._jobstatus_path), {}, lock=True)
status.update(params)
store.save_mk_file(str(self._jobstatus_path), self._format_value(status))
finally:
store.release_lock(str(self._jobstatus_path))
def test_release_lock(tmpdir):
locked_file = tmpdir.join("locked_file")
locked_file.write("")
path = "%s" % locked_file
assert store.have_lock(path) is False
store.aquire_lock(path)
assert store.have_lock(path) is True
store.release_lock(path)
assert store.have_lock(path) is False
def test_release_lock_already_closed(locked_file, path_type):
path = path_type(locked_file)
assert store.have_lock(path) is False
store.aquire_lock(path)
assert store.have_lock(path) is True
os.close(store._locks._get_lock(str(path)))
store.release_lock(path)
assert store.have_lock(path) is False
def _cleanup_locked_pid_file(path: Path) -> None:
"""Cleanup the lock + file acquired by the function above"""
if not store.have_lock(str(path)):
return
store.release_lock(str(path))
try:
path.unlink()
except OSError:
pass
def load(self, hostname: HostName) -> None:
filename = cmk.utils.paths.counters_dir + "/" + hostname
try:
# TODO: refactoring. put these two values into a named tuple
self._item_states = store.load_object_from_file(
filename,
default={},
lock=True,
)
self._last_mtime = os.stat(filename).st_mtime
finally:
store.release_lock(filename)
def test_release_lock_already_closed(locked_file, path_type):
path = path_type(locked_file)
assert store.have_lock(path) is False
store.aquire_lock(path)
assert store.have_lock(path) is True
fd = store._locks._get_lock(str(path))
assert isinstance(fd, int)
os.close(fd)
store.release_lock(path)
assert store.have_lock(path) is False
def test_release_lock_already_closed(tmp_path):
locked_file = tmp_path / "locked_file"
locked_file.write_text(u"", encoding="utf-8")
path = str(locked_file)
assert store.have_lock(path) is False
store.aquire_lock(path)
assert store.have_lock(path) is True
os.close(store._acquired_locks[path])
store.release_lock(path)
assert store.have_lock(path) is False
def test_release_lock_already_closed(tmpdir):
locked_file = tmpdir.join("locked_file")
locked_file.write("")
path = "%s" % locked_file
assert store.have_lock(path) is False
store.aquire_lock(path)
assert store.have_lock(path) is True
os.close(store._acquired_locks[path])
store.release_lock(path)
assert store.have_lock(path) is False
def save_change(self, change_spec):
path = self._site_changes_path()
try:
store.aquire_lock(path)
with open(path, "a+") as f:
f.write(repr(change_spec) + "\0")
f.flush()
os.fsync(f.fileno())
os.chmod(path, 0660)
except Exception as e:
raise MKGeneralException(_("Cannot write file \"%s\": %s") % (path, e))
finally:
store.release_lock(path)
def save_change(self, change_spec):
path = self._site_changes_path()
try:
store.aquire_lock(path)
with path.open("ab+") as f:
f.write(six.ensure_binary(repr(change_spec)) + b"\0")
f.flush()
os.fsync(f.fileno())
path.chmod(0o660)
except Exception as e:
raise MKGeneralException(_("Cannot write file \"%s\": %s") % (path, e))
finally:
store.release_lock(path)
def append(self, entry: _VT) -> None:
path = self._path
try:
store.aquire_lock(path)
with path.open("ab+") as f:
f.write(repr(entry).encode("utf-8") + b"\0")
f.flush()
os.fsync(f.fileno())
path.chmod(0o660)
except Exception as e:
raise MKGeneralException(_("Cannot write file \"%s\": %s") % (path, e))
finally:
store.release_lock(path)
def disksync(
self,
*,
removed: Container[_TKey] = (),
updated: Iterable[Tuple[_TKey, _TValue]] = (),
) -> None:
"""Re-load and write the changes of the stored values
This method will reload the values from disk, apply the changes (remove keys
and update values) as specified by the arguments, and then write the result to disk.
When this method returns, the data provided via the Mapping-interface and
the data stored on disk must be in sync.
"""
self._log_debug("synchronizing")
self._path.parent.mkdir(parents=True, exist_ok=True)
try:
store.aquire_lock(self._path)
if self._path.stat().st_mtime == self._last_sync:
self._log_debug("already loaded")
else:
self._log_debug("loading from disk")
self._data = self._deserializer(
store.load_text_from_file(self._path,
default="{}",
lock=False))
if removed or updated:
data = {
k: v
for k, v in self._data.items() if k not in removed
}
data.update(updated)
self._log_debug("writing to disk")
store.save_text_to_file(self._path, self._serializer(data))
self._data = data
self._last_sync = self._path.stat().st_mtime
except Exception as exc:
raise MKGeneralException from exc
finally:
store.release_lock(self._path)
def get_status_from_file(self) -> JobStatusSpec:
if not self._jobstatus_path.exists():
data: JobStatusSpec = {}
data["state"] = JobStatusStates.INITIALIZED
data["started"] = time.time()
else:
try:
# Read this data with an explicit lock
# This prevents a race condition where an empty jobstatus.mk file is read
data = store.load_object_from_file(str(self._jobstatus_path),
default={},
lock=True)
# Repair broken/invalid files
if "state" not in data:
data["state"] = JobStatusStates.INITIALIZED
data["started"] = os.path.getctime(
str(self._jobstatus_path))
finally:
store.release_lock(str(self._jobstatus_path))
data.setdefault("pid", None)
data["loginfo"] = {}
for field_id, field_path in [
("JobProgressUpdate", self._progress_update_path),
("JobResult", self._result_message_path),
("JobException", self._exceptions_path),
]:
if field_path.exists():
with field_path.open(encoding="utf-8") as f:
data["loginfo"][field_id] = f.read().splitlines()
else:
data["loginfo"][field_id] = []
map_substate_to_active = {
JobStatusStates.INITIALIZED: True,
JobStatusStates.RUNNING: True,
JobStatusStates.FINISHED: False,
JobStatusStates.STOPPED: False,
JobStatusStates.EXCEPTION: False,
}
data["is_active"] = map_substate_to_active[data["state"]]
return data
def disksync(
self,
*,
removed: Container[_ValueStoreKey] = (),
updated: Iterable[Tuple[_ValueStoreKey, Any]] = (),
) -> None:
"""Re-load and then store the changes of the item state to disk
Make sure the object is in sync with the file after writing.
"""
self._log_debug("value store: synchronizing")
self._path.parent.mkdir(parents=True, exist_ok=True)
try:
store.aquire_lock(self._path)
if self._path.stat().st_mtime == self._last_sync:
self._log_debug("value store: already loaded")
else:
self._log_debug("value store: loading from disk")
self._data = store.load_object_from_file(self._path,
default={},
lock=False)
if removed or updated:
data = {
k: v
for k, v in self._data.items() if k not in removed
}
data.update(updated)
self._log_debug("value store: writing to disk")
store.save_object_to_file(self._path, data, pretty=False)
self._data = data
self._last_sync = self._path.stat().st_mtime
except Exception as exc:
raise MKGeneralException from exc
finally:
store.release_lock(self._path)
def save(self, hostname):
# type: (HostName) -> None
""" The job of the save function is to update the item state on disk.
It simply returns, if it detects that the data wasn't changed at all since the last loading
If the data on disk has been changed in the meantime, the cached data is updated from disk.
Afterwards only the actual modifications (update/remove) are applied to the updated cached
data before it is written back to disk.
"""
filename = cmk.utils.paths.counters_dir + "/" + hostname
if not self._removed_item_state_keys and not self._updated_item_states:
return
try:
if not os.path.exists(cmk.utils.paths.counters_dir):
os.makedirs(cmk.utils.paths.counters_dir)
store.aquire_lock(filename)
last_mtime = os.stat(filename).st_mtime
if last_mtime != self._last_mtime:
self._item_states = store.load_object_from_file(filename,
default={})
# Remove obsolete keys
for key in self._removed_item_state_keys:
try:
del self._item_states[key]
except KeyError:
pass
# Add updated keys
self._item_states.update(self._updated_item_states)
store.save_object_to_file(filename,
self._item_states,
pretty=False)
except Exception:
raise MKGeneralException("Cannot write to %s: %s" %
(filename, traceback.format_exc()))
finally:
store.release_lock(filename)
