def get_status_from_file(self) -> JobStatusSpec:
if not self._jobstatus_path.exists():
data: JobStatusSpec = {}
data["state"] = JobStatusStates.INITIALIZED
data["started"] = time.time()
else:
try:
# Read this data with an explicit lock
# This prevents a race condition where an empty jobstatus.mk file is read
data = store.load_object_from_file(str(self._jobstatus_path),
default={},
lock=True)
# Repair broken/invalid files
if "state" not in data:
data["state"] = JobStatusStates.INITIALIZED
data["started"] = os.path.getctime(
str(self._jobstatus_path))
finally:
store.release_lock(str(self._jobstatus_path))
data.setdefault("pid", None)
data["loginfo"] = {}
for field_id, field_path in [
("JobProgressUpdate", self._progress_update_path),
("JobResult", self._result_message_path),
("JobException", self._exceptions_path),
]:
if field_path.exists():
with field_path.open(encoding="utf-8") as f:
data["loginfo"][field_id] = f.read().splitlines()
else:
data["loginfo"][field_id] = []
map_substate_to_active = {
JobStatusStates.INITIALIZED: True,
JobStatusStates.RUNNING: True,
JobStatusStates.FINISHED: False,
JobStatusStates.STOPPED: False,
JobStatusStates.EXCEPTION: False,
}
data["is_active"] = map_substate_to_active[data["state"]]
return data
def get_gui_messages(user_id=None):
if user_id is None:
user_id = config.user.id
path = config.config_dir + "/" + ensure_str(user_id) + '/messages.mk'
messages = store.load_object_from_file(path, default=[])
# Delete too old messages
updated = False
for index, message in enumerate(messages):
now = time.time()
valid_till = message.get('valid_till')
if valid_till is not None and valid_till < now:
messages.pop(index)
updated = True
if updated:
save_gui_messages(messages)
return messages
def get_gui_messages(user_id=None):
if user_id is None:
user_id = user.id
path = cmk.utils.paths.profile_dir / user_id / "messages.mk"
messages = store.load_object_from_file(path, default=[])
# Delete too old messages
updated = False
for index, message in enumerate(messages):
now = time.time()
valid_till = message.get("valid_till")
if valid_till is not None and valid_till < now:
messages.pop(index)
updated = True
if updated:
save_gui_messages(messages)
return messages
def save(self, hostname):
# type: (HostName) -> None
""" The job of the save function is to update the item state on disk.
It simply returns, if it detects that the data wasn't changed at all since the last loading
If the data on disk has been changed in the meantime, the cached data is updated from disk.
Afterwards only the actual modifications (update/remove) are applied to the updated cached
data before it is written back to disk.
"""
filename = cmk.utils.paths.counters_dir + "/" + hostname
if not self._removed_item_state_keys and not self._updated_item_states:
return
try:
if not os.path.exists(cmk.utils.paths.counters_dir):
os.makedirs(cmk.utils.paths.counters_dir)
store.aquire_lock(filename)
last_mtime = os.stat(filename).st_mtime
if last_mtime != self._last_mtime:
self._item_states = store.load_object_from_file(filename,
default={})
# Remove obsolete keys
for key in self._removed_item_state_keys:
try:
del self._item_states[key]
except KeyError:
pass
# Add updated keys
self._item_states.update(self._updated_item_states)
store.save_object_to_file(filename,
self._item_states,
pretty=False)
except Exception:
raise MKGeneralException("Cannot write to %s: %s" %
(filename, traceback.format_exc()))
finally:
store.release_lock(filename)
def disksync(
self,
*,
removed: Container[_ValueStoreKey] = (),
updated: Iterable[Tuple[_ValueStoreKey, Any]] = (),
) -> None:
"""Re-load and then store the changes of the item state to disk
Make sure the object is in sync with the file after writing.
"""
self._log_debug("value store: synchronizing")
self._path.parent.mkdir(parents=True, exist_ok=True)
try:
store.aquire_lock(self._path)
if self._path.stat().st_mtime == self._last_sync:
self._log_debug("value store: already loaded")
else:
self._log_debug("value store: loading from disk")
self._data = store.load_object_from_file(self._path,
default={},
lock=False)
if removed or updated:
data = {
k: v
for k, v in self._data.items() if k not in removed
}
data.update(updated)
self._log_debug("value store: writing to disk")
store.save_object_to_file(self._path, data, pretty=False)
self._data = data
self._last_sync = self._path.stat().st_mtime
except Exception as exc:
raise MKGeneralException from exc
finally:
store.release_lock(self._path)
def load(
self,
*,
trees: Iterable[BackendSNMPTree],
) -> None:
"""Try to read the OIDs data from cache files"""
for tree in trees:
for oid in (o for o in tree.oids if o.save_to_cache): # no point in reading otherwise
fetchoid = f"{tree.base}.{oid.column}"
path = self._path / fetchoid
console.vverbose(f" Loading {fetchoid} from walk cache {path}\n")
try:
read_walk = store.load_object_from_file(path)
except Exception:
console.verbose(f" Failed to load {fetchoid} from walk cache {path}\n")
if cmk.utils.debug.enabled():
raise
continue
if read_walk is not None:
self._store[fetchoid] = (oid.save_to_cache, read_walk) # (True, ...)
def get_history_deltas(hostname, search_timestamp=None):
if '/' in hostname:
return None, [] # just for security reasons
inventory_path = "%s/inventory/%s" % (cmk.utils.paths.var_dir, hostname)
if not os.path.exists(inventory_path):
return [], []
latest_timestamp = str(int(os.stat(inventory_path).st_mtime))
inventory_archive_dir = "%s/inventory_archive/%s" % (
cmk.utils.paths.var_dir, hostname)
try:
archived_timestamps = sorted(os.listdir(inventory_archive_dir))
except OSError:
return [], []
all_timestamps = archived_timestamps + [latest_timestamp]
previous_timestamp = None
if not search_timestamp:
required_timestamps = all_timestamps
else:
new_timestamp_idx = all_timestamps.index(search_timestamp)
if new_timestamp_idx == 0:
required_timestamps = [search_timestamp]
else:
previous_timestamp = all_timestamps[new_timestamp_idx - 1]
required_timestamps = [search_timestamp]
tree_lookup = {}
def get_tree(timestamp):
if timestamp is None:
return StructuredDataTree()
if timestamp in tree_lookup:
return tree_lookup[timestamp]
if timestamp == latest_timestamp:
inventory_tree = load_filtered_inventory_tree(hostname)
if inventory_tree is None:
return
tree_lookup[timestamp] = inventory_tree
else:
inventory_archive_path = "%s/%s" % (inventory_archive_dir,
timestamp)
tree_lookup[timestamp] = _filter_tree(
StructuredDataTree().load_from(inventory_archive_path))
return tree_lookup[timestamp]
corrupted_history_files = []
delta_history = []
for _idx, timestamp in enumerate(required_timestamps):
cached_delta_path = os.path.join(
cmk.utils.paths.var_dir, "inventory_delta_cache", hostname,
"%s_%s" % (previous_timestamp, timestamp))
cached_data = None
try:
cached_data = store.load_object_from_file(cached_delta_path)
except MKGeneralException:
pass
if cached_data:
new, changed, removed, delta_tree_data = cached_data
delta_tree = StructuredDataTree()
delta_tree.create_tree_from_raw_tree(delta_tree_data)
delta_history.append(
(timestamp, (new, changed, removed, delta_tree)))
previous_timestamp = timestamp
continue
try:
previous_tree = get_tree(previous_timestamp)
current_tree = get_tree(timestamp)
delta_data = current_tree.compare_with(previous_tree)
new, changed, removed, delta_tree = delta_data
if new or changed or removed:
store.save_file(
cached_delta_path,
repr((new, changed, removed, delta_tree.get_raw_tree())),
)
delta_history.append((timestamp, delta_data))
except RequestTimeout:
raise
except LoadStructuredDataError:
corrupted_history_files.append(
str(get_short_inventory_history_filepath(hostname, timestamp)))
previous_timestamp = timestamp
return delta_history, corrupted_history_files
def load_from(self, filepath):
raw_tree = store.load_object_from_file(filepath)
return self.create_tree_from_raw_tree(raw_tree)
def _load_ip_lookup_cache(lock):
# type: (bool) -> NewIPLookupCache
return _convert_legacy_ip_lookup_cache(
store.load_object_from_file(_cache_path(), default={}, lock=lock))
def load_acknowledgements():
return store.load_object_from_file(acknowledgement_path, default=[])
def _load_activation(self):
self.__dict__.update(store.load_object_from_file(self._info_path(), {}))
def _load_site_replication_status(site_id, lock=False):
return store.load_object_from_file(
_site_replication_status_path(site_id),
default={},
lock=lock,
)
def load_file(self, name: str, deflt: Any, lock: bool = False) -> Any:
if self.confdir is None:
return deflt
path = self.confdir + "/" + name + ".mk"
return store.load_object_from_file(path, default=deflt, lock=lock)
def load_config(self) -> None:
if not Path(self._bi_configuration_file).exists():
self._load_config(bi_sample_config)
return
self._load_config(
store.load_object_from_file(self._bi_configuration_file))
def _do_load_legacy_bookmarks(cls):
path = config.user.confdir + "/bookmarks.mk"
return store.load_object_from_file(path, default=[])
def load(self) -> PersistedSections[TRawDataSection]:
raw_sections_data = _store.load_object_from_file(self.path, default={})
return PersistedSections[TRawDataSection](
{SectionName(k): v
for k, v in raw_sections_data.items()})
def _load_single_oid_cache(snmp_config):
# type: (SNMPHostConfig) -> Dict[OID, Optional[RawValue]]
cache_path = "%s/%s.%s" % (cmk.utils.paths.snmp_scan_cache_dir,
snmp_config.hostname, snmp_config.ipaddress)
return store.load_object_from_file(cache_path, default={})
def _load_single_oid_cache(
snmp_config: SNMPHostConfig) -> Dict[OID, Optional[SNMPDecodedString]]:
cache_path = "%s/%s.%s" % (cmk.utils.paths.snmp_scan_cache_dir,
snmp_config.hostname, snmp_config.ipaddress)
return store.load_object_from_file(cache_path, default={})
def _last_activation_state(self, site_id):
manager = ActivateChangesManager()
site_state_path = os.path.join(manager.activation_persisted_dir,
manager.site_filename(site_id))
return store.load_object_from_file(site_state_path, {})
def _load_newest_host_labels_per_site(self) -> Dict[SiteId, float]:
return store.load_object_from_file(
DiscoveredHostLabelSyncJob.newest_host_labels_per_site_path(), default={})
def _load_site_state(self, site_id):
return store.load_object_from_file(self.site_state_path(site_id), {})
def _read(self, file_path: Path) -> HostsData:
return store.load_object_from_file(str(file_path), default={})
def get_history_deltas(
hostname: HostName,
search_timestamp: Optional[str] = None
) -> Tuple[List[Tuple[str, InventoryDeltaData]], List[str]]:
if '/' in hostname:
return [], [] # just for security reasons
inventory_path = "%s/%s" % (cmk.utils.paths.inventory_output_dir, hostname)
if not os.path.exists(inventory_path):
return [], []
latest_timestamp = str(int(os.stat(inventory_path).st_mtime))
inventory_archive_dir = "%s/%s" % (cmk.utils.paths.inventory_archive_dir,
hostname)
try:
archived_timestamps = sorted(os.listdir(inventory_archive_dir))
except OSError:
return [], []
all_timestamps: List[str] = archived_timestamps + [latest_timestamp]
previous_timestamp: Optional[str] = None
if not search_timestamp:
required_timestamps = all_timestamps
else:
new_timestamp_idx = all_timestamps.index(search_timestamp)
if new_timestamp_idx == 0:
required_timestamps = [search_timestamp]
else:
previous_timestamp = all_timestamps[new_timestamp_idx - 1]
required_timestamps = [search_timestamp]
tree_lookup: Dict[str, Any] = {}
def get_tree(timestamp: Optional[str]) -> StructuredDataNode:
if timestamp is None:
return StructuredDataNode()
if timestamp in tree_lookup:
return tree_lookup[timestamp]
if timestamp == latest_timestamp:
inventory_tree = load_filtered_inventory_tree(hostname)
if inventory_tree is None:
raise LoadStructuredDataError()
tree_lookup[timestamp] = inventory_tree
else:
inventory_archive_path = Path(inventory_archive_dir, timestamp)
tree_lookup[timestamp] = _filter_tree(
StructuredDataStore.load_file(inventory_archive_path))
return tree_lookup[timestamp]
corrupted_history_files = []
delta_history: List[Tuple[str, InventoryDeltaData]] = []
for _idx, timestamp in enumerate(required_timestamps):
cached_delta_path = os.path.join(
cmk.utils.paths.var_dir, "inventory_delta_cache", hostname,
"%s_%s" % (previous_timestamp, timestamp))
cached_data = None
try:
cached_data = store.load_object_from_file(cached_delta_path,
default=None)
except MKGeneralException:
pass
if cached_data:
new, changed, removed, delta_tree_data = cached_data
delta_tree = StructuredDataNode.deserialize(delta_tree_data)
delta_history.append(
(timestamp, (new, changed, removed, delta_tree)))
previous_timestamp = timestamp
continue
try:
previous_tree = get_tree(previous_timestamp)
current_tree = get_tree(timestamp)
delta_result = current_tree.compare_with(previous_tree)
delta_data = (delta_result.counter['new'],
delta_result.counter['changed'],
delta_result.counter['removed'], delta_result.delta)
new, changed, removed, delta_tree = delta_data
if new or changed or removed:
store.save_text_to_file(
cached_delta_path,
repr((new, changed, removed, delta_tree.serialize())),
)
delta_history.append((timestamp, delta_data))
except LoadStructuredDataError:
corrupted_history_files.append(
str(get_short_inventory_history_filepath(hostname, timestamp)))
previous_timestamp = timestamp
return delta_history, corrupted_history_files
def test_save_data_to_file(tmp_path, path_type, data):
path = path_type(tmp_path / "lala")
store.save_object_to_file(path, data)
assert store.load_object_from_file(path, default=None) == data
def _load_result(path: Path) -> Union[str, object]:
if remote_automation_call_came_from_pre21():
return store.load_object_from_file(path, default=None)
return store.load_text_from_file(path)
def test_load_data_from_file_empty(tmp_path, path_type):
locked_file = tmp_path / "test"
locked_file.write_text("", encoding="utf-8")
data = store.load_object_from_file(path_type(tmp_path / "x"),
default="DEF")
assert data == "DEF"
def _do_load_legacy_bookmarks(cls):
if config.user.confdir is None:
raise Exception("user confdir is None")
path = config.user.confdir + "/bookmarks.mk"
return store.load_object_from_file(path, default=[])
def test_load_data_not_locked(tmp_path, path_type):
locked_file = tmp_path / "locked_file"
locked_file.write_text("[1, 2]", encoding="utf-8")
store.load_object_from_file(path_type(locked_file), default=None)
assert store.have_lock(path_type(locked_file)) is False
def _load_acknowledgements(self, lock=False):
return store.load_object_from_file(self._ack_path,
default={},
lock=lock)
def pid_from_file(pid_file: Path) -> Optional[ProcessId]:
"""Read a process id from a given pid file"""
try:
return ProcessId(int(load_object_from_file(pid_file, default=None)))
except Exception:
return None
