def _handleCentralStorage(self,
file_data: str,
plugin_path: str,
is_bundled_plugin: bool = False) -> bool:
"""
Plugins can indicate that they want certain things to be stored in a central location.
In the case of a signed plugin you *must* do this by means of the central_storage.json file.
:param file_data: The data as loaded from the file
:param plugin_path: The location of the plugin on the file system
:return: False if there is a security suspicion, True otherwise (even if the method otherwise fails).
"""
try:
file_manifest = json.loads(file_data)
except (json.decoder.JSONDecodeError, UnicodeDecodeError):
Logger.logException(
"e",
f"Failed to parse the central storage file for '{plugin_path}'."
)
return True
for file_to_move in file_manifest:
full_path = os.path.join(plugin_path, file_to_move[0])
try:
CentralFileStorage.store(full_path,
file_to_move[1],
Version(file_to_move[2]),
move_file=not is_bundled_plugin)
except (FileExistsError, TypeError, IndexError, OSError):
Logger.logException(
"w",
f"Can't move file {file_to_move[0]} to central storage for '{plugin_path}'."
)
return True
def test_retrieveWrongHash():
"""
Tests retrieving a file that has a wrong hash.
"""
with unittest.mock.patch("UM.Resources.Resources.getDataStoragePath", lambda: "test_central_storage/4.9"):
CentralFileStorage.store(TEST_FILE_PATH, "myfile")
pytest.raises(IOError, lambda: CentralFileStorage.retrieve("myfile", TEST_FILE_HASH2))
def test_storeConflict():
"""
Tests storing two different files under the same ID/version.
"""
with unittest.mock.patch("UM.Resources.Resources.getDataStoragePath", lambda: "test_central_storage/4.9"):
CentralFileStorage.store(TEST_FILE_PATH, "myfile")
pytest.raises(FileExistsError, lambda: CentralFileStorage.store(TEST_FILE_PATH2, "myfile"))
def test_storeNonExistent():
"""
Tests storing a file that doesn't exist.
The storage is expected to fail silently, leaving just a debug statement that it was already stored.
"""
with unittest.mock.patch("UM.Resources.Resources.getDataStoragePath", lambda: "test_central_storage/4.9"):
CentralFileStorage.store("non_existent_file.txt", "my_non_existent_file") # Shouldn't raise error.
def test_storeRetrieve():
"""
Basic store-retrieve loop test.
"""
with unittest.mock.patch("UM.Resources.Resources.getDataStoragePath", lambda: "test_central_storage/4.9"):
CentralFileStorage.store(TEST_FILE_PATH, "myfile")
stored_path = CentralFileStorage.retrieve("myfile", TEST_FILE_HASH)
assert not os.path.exists(TEST_FILE_PATH)
assert os.path.exists(stored_path)
assert open(stored_path, "rb").read() == TEST_FILE_CONTENTS
def test_storeDuplicate():
"""
Tests storing a file twice.
The storage should make the files unique, i.e. remove the duplicate.
"""
with unittest.mock.patch("UM.Resources.Resources.getDataStoragePath", lambda: "test_central_storage/4.9"):
shutil.copy(TEST_FILE_PATH, TEST_FILE_PATH + ".copy.txt")
CentralFileStorage.store(TEST_FILE_PATH, "myfile")
CentralFileStorage.store(TEST_FILE_PATH + ".copy.txt", "myfile") # Shouldn't raise error. File contents are identical.
assert not os.path.exists(TEST_FILE_PATH + ".copy.txt") # Duplicate must be removed.
def init_trust(self):
# Create a temporary directory and save a test key-pair to it:
temp_dir = tempfile.TemporaryDirectory()
temp_path = temp_dir.name
private_key, public_key = TrustBasics.generateNewKeyPair()
private_path = os.path.join(temp_path, "test_private_key.pem")
public_path = os.path.join(temp_path, "test_public_key.pem")
TrustBasics.saveKeyPair(private_key, private_path, public_path,
_passphrase)
# Create random files:
all_paths = [
os.path.abspath(os.path.join(temp_path, x, y, z))
for x in _folder_names for y in _subfolder_names
for z in _file_names
]
for path in all_paths:
folder_path = os.path.dirname(path)
if not os.path.exists(folder_path):
os.makedirs(folder_path)
with open(path, "w") as file:
file.write("".join(
random.choice(['a', 'b', 'c', '0', '1', '2', '\n'])
for _ in range(1024)))
# Set up mocked Central File Storage & plugin file (don't move the files yet, though):
CentralFileStorage.setIsEnterprise(True)
central_storage_dir = tempfile.TemporaryDirectory()
central_storage_path = central_storage_dir.name
large_plugin_path = os.path.join(temp_path, _folder_names[2])
store_folder = os.path.join(large_plugin_path, _subfolder_names[0])
store_file = os.path.join(large_plugin_path, _file_names[2])
central_storage_dict = [[
f"{_subfolder_names[0]}", f"{_subfolder_names[0]}", "1.0.0",
CentralFileStorage._hashItem(store_folder)
],
[
f"{_file_names[2]}", f"{_file_names[2]}",
"1.0.0",
CentralFileStorage._hashItem(store_file)
]]
central_storage_file_path = os.path.join(
large_plugin_path, TrustBasics.getCentralStorageFilename())
with open(central_storage_file_path, "w") as file:
json.dump(central_storage_dict, file, indent=2)
# Instantiate a trust object with the public key that was just generated:
violation_callback = MagicMock()
trust = Trust(
public_path) # No '.getInstance', since key & handler provided.
trust._violation_handler = violation_callback
yield temp_path, private_path, trust, violation_callback, central_storage_path
temp_dir.cleanup()
central_storage_dir.cleanup()
CentralFileStorage.setIsEnterprise(False)
def test_storeVersions():
with unittest.mock.patch("UM.Resources.Resources.getDataStoragePath", lambda: "test_central_storage/4.9"):
CentralFileStorage.store(TEST_FILE_PATH, "myfile", Version("1.0.0"))
CentralFileStorage.store(TEST_FILE_PATH2, "myfile", Version("1.1.0"))
stored_path1 = CentralFileStorage.retrieve("myfile", TEST_FILE_HASH, Version("1.0.0"))
stored_path2 = CentralFileStorage.retrieve("myfile", TEST_FILE_HASH2, Version("1.1.0"))
assert stored_path1 != stored_path2
assert open(stored_path1, "rb").read() == TEST_FILE_CONTENTS
assert open(stored_path2, "rb").read() == TEST_FILE_CONTENTS2
def test_signFolderAndVerify(self, init_trust):
temp_dir, private_path, trust_instance, violation_callback, central_storage_dir = init_trust
folderpath_signed = os.path.join(temp_dir, _folder_names[0])
folderpath_unsigned = os.path.join(temp_dir, _folder_names[1])
folderpath_large = os.path.join(temp_dir, _folder_names[2])
# Attempt to sign a folder & validate it's signatures.
assert signFolder(private_path, folderpath_signed, [], _passphrase)
assert trust_instance.signedFolderCheck(folderpath_signed)
# A folder that is not signed should be seen as such
assert not trust_instance.signedFolderCheck(folderpath_unsigned)
assert violation_callback.call_count == 1
violation_callback.reset_mock()
# Unknown folders should also be seen as unsigned
assert not trust_instance.signedFileCheck("folder-not-found-check")
assert violation_callback.call_count == 1
violation_callback.reset_mock()
# After removing the key, the folder that was signed should be seen as unsigned.
public_key = copy.copy(trust_instance._public_key)
trust_instance._public_key = None
assert not trust_instance.signedFolderCheck(folderpath_signed)
assert violation_callback.call_count == 1
violation_callback.reset_mock()
trust_instance._public_key = public_key
# Hecking around with the signature file should also be discouraged.
signatures_file = os.path.join(
folderpath_signed, TrustBasics.getSignaturesLocalFilename())
with open(signatures_file, "r", encoding="utf-8") as sigfile:
sig_json = json.load(sigfile)
restore_json = copy.copy(sig_json)
sig_json[TrustBasics.getRootSignedManifestKey()] = "HAHAHAHA"
os.remove(signatures_file)
with open(signatures_file, "w", encoding="utf-8") as sigfile:
json.dump(sig_json, sigfile, indent=2)
assert not trust_instance.signedFolderCheck(folderpath_signed)
assert violation_callback.call_count > 0
violation_callback.reset_mock()
os.remove(signatures_file)
with open(signatures_file, "w", encoding="utf-8") as sigfile:
json.dump(restore_json, sigfile, indent=2)
# Any modification should also invalidate it.
filepath = os.path.join(folderpath_signed, _subfolder_names[0],
_file_names[1])
with open(filepath, "w") as file:
file.write("\nAlice and Bob will never notice this! Hehehehe.\n")
assert not trust_instance.signedFolderCheck(folderpath_signed)
assert violation_callback.call_count > 0
violation_callback.reset_mock()
# Any missing files should also be registered.
os.remove(filepath)
assert not trust_instance.signedFolderCheck(folderpath_signed)
assert violation_callback.call_count == 1
violation_callback.reset_mock()
# * 'Central file storage'-enabled section *
with patch(
"UM.CentralFileStorage.CentralFileStorage.getCentralStorageLocation",
MagicMock(return_value=central_storage_dir)):
# Do some set-up (signing, moving files around with the central file storage):
assert signFolder(private_path, folderpath_large, [], _passphrase)
assert violation_callback.call_count == 0
subfolder_path = os.path.join(folderpath_large,
_subfolder_names[0])
file_path = os.path.join(folderpath_large, _file_names[2])
stored_file_path = os.path.join(central_storage_dir,
_file_names[2] + ".1.0.0")
CentralFileStorage.store(subfolder_path, _subfolder_names[0],
"1.0.0", True)
CentralFileStorage.store(file_path, _file_names[2], "1.0.0", True)
# Should pass signed folder check, even though files have moved to central storage:
assert trust_instance.signedFolderCheck(folderpath_large)
assert violation_callback.call_count == 0
# Should not pass the signed folder check if one of the files doesn't have the right hash in storage.
with open(stored_file_path, "w") as file:
file.write(
"\nWhoopsadoodle, the file just changed suddenly.\n")
assert not trust_instance.signedFolderCheck(folderpath_large)
assert violation_callback.call_count > 0
violation_callback.reset_mock()
# Should not pass the signed folder check if one of the files is missing, even in storage.
os.remove(stored_file_path)
assert not trust_instance.signedFolderCheck(folderpath_large)
assert violation_callback.call_count > 0
violation_callback.reset_mock()
def test_retrieveNonExistent():
"""
Tests retrieving a file that is not stored in the central location.
"""
with unittest.mock.patch("UM.Resources.Resources.getDataStoragePath", lambda: "test_central_storage/4.9"):
pytest.raises(FileNotFoundError, lambda: CentralFileStorage.retrieve("non_existent_file", "0123456789ABCDEF"))
def signedFolderCheck(self, path: str) -> bool:
"""In the 'singed folder' case, check whether the folder is signed according to the Trust-objects' public key.
:param path: The path to the folder to be checked (not the signature-file).
:return: True if the folder is signed (contains a signatures-file) and signed correctly.
"""
try:
manifest_path = os.path.join(path, TrustBasics.getSignaturesLocalFilename())
storage_filename = os.path.join(path, TrustBasics.getCentralStorageFilename())
storage_list = None
if os.path.exists(storage_filename):
with open(storage_filename, "r", encoding = "utf-8") as storage_file:
storage_list = json.load(storage_file)
# Open the file containing signatures:
with open(manifest_path, "r", encoding = "utf-8") as manifest_file:
manifest_content = json.load(manifest_file)
file_signatures = manifest_content.get(TrustBasics.getRootSignatureCategory(), None)
if file_signatures is None:
self._violation_handler("Can't parse (folder) signature file '{0}'.".format(manifest_file))
return False
# Any filename outside of the plugin-root is a sure sign of tampering:
for key in file_signatures.keys():
if ".." in key:
self._violation_handler("Suspect key '{0}' in signature file '{1}'.".format(key, manifest_file))
return False
# Check if the signing file itself has been tampered with (manifest is self-signed):
if not self._verifyManifestIntegrety(file_signatures, manifest_content):
self._violation_handler(f"Manifest '{manifest_file}' is not properly self-signed in '{path}'.")
return False
# Loop over all files within the folder (excluding the signature file):
file_count = 0
for root, dirnames, filenames in os.walk(path, followlinks = self._follow_symlinks):
for filename in filenames:
if filename == TrustBasics.getSignaturesLocalFilename() and root == path:
continue
name_on_disk, name_in_data = TrustBasics.getFilePathInfo(path, root, filename)
file_count += 1
# Get the signature for the current to-verify file:
signature = file_signatures.get(name_in_data, None)
if signature is None:
self._violation_handler("File '{0}' was not signed with a checksum.".format(name_on_disk))
return False
# Verify the file:
if not self._verifyFile(name_on_disk, signature):
self._violation_handler("File '{0}' didn't match with checksum.".format(name_on_disk))
return False
for dirname in dirnames:
dir_full_path = os.path.join(path, dirname)
if os.path.islink(dir_full_path) and not self._follow_symlinks:
Logger.log("w", "Directory symbolic link '{0}' will not be followed.".format(dir_full_path))
# Check if the files moved to storage are still correct.
if storage_list:
for entry in storage_list:
try:
# If this doesn't raise an exception, it's correct, since central storage uses hashes.
central_storage_path = CentralFileStorage.retrieve(entry[1], entry[3], Version(entry[2]))
# File could have been removed during execution (also helps with tests).
if not os.path.exists(central_storage_path):
continue
# If a directory was moved, add all the files in that directory to the file_count. For
# individual files mentioned in the central_storage.json increment the file_count by 1.
if os.path.isdir(central_storage_path):
file_count += sum([len(files) for _, _, files in os.walk(central_storage_path)])
elif os.path.isfile(central_storage_path):
file_count += 1
except (EnvironmentError, IOError):
self._violation_handler(f"Couldn't verify at least one centrally stored file for '{path}'.")
return False
# The number of correctly signed files should be the same as the number of signatures:
if len(file_signatures.keys()) != file_count:
self._violation_handler("Mismatch: # entries in '{0}' vs. real files.".format(manifest_path))
return False
Logger.log("i", "Verified unbundled folder '{0}'.".format(path))
return True
except: # Yes, we do really want this on _every_ exception that might occur.
self._violation_handler(f"Exception during verification of unbundled folder '{path}'.")
return False
def signedFolderPreStorageCheck(self, path: str) -> bool:
"""Do a quick check whether the 'central storage file' of a folder has been tampered with. This is necessary,
since the central storage system (which otherwise runs first) copies files, and the copying of files itself
can be an attack. Note that right after copying, a full check can be done, so the files themselves don't have
to be checked yet (since that happens in the full check after copying).
Shared pools of versioned items ('central storage') are used if a folder contains a 'central storage file'.
(See the CentralFileStorage class for details.) The 'canonical' version of a folder as far as the Trust system
is concerned, is the one where the items are already in central storage. Otherwise there would either be a whole
range of 'acceptable answers' (and that's harder to test against) or, there would always be a need to verify
a situation that shouldn't be that frequent (items need to be copied to central storage). This creates a problem
in that the central storage mechanism needs to run _first_, however. The central storage has its own security
measures, but this means that the central storage file in a folder (which contains info on what items should be
copied) hasn't been checked against the trust manifest file in that folder yet.
This only concerns the signature of the central storage file (and the correctness of the manifest file itself).
Per separation of concern, and since the storage system already needs to be aware of security, any other
'sanity checks' on the contents central storage file itself are the job of that system.
:param path: The folder to do a quick pre-move check for.
:return: True if the central-storage file is correctly signed. A folder without such a file is correct as well.
"""
try:
# Check if the central storage file exist, if not, then the system won't copy anything in any case.
central_storage_filename = os.path.join(path, TrustBasics.getCentralStorageFilename())
if not os.path.exists(central_storage_filename):
Logger.log("i", f"No central storage file for unbundled folder '{path}'.")
return True
# Open the file containing signatures (just reading the json is negligible compared to the verify or store):
manifest_path = os.path.join(path, TrustBasics.getSignaturesLocalFilename())
with open(manifest_path, "r", encoding = "utf-8") as manifest_file:
manifest_content = json.load(manifest_file)
file_signatures = manifest_content.get(TrustBasics.getRootSignatureCategory(), None)
if file_signatures is None:
self._violation_handler(f"Can't parse (folder) signature file '{manifest_file}' in '{path}'.")
return False
# Check if there is an entry, since this file is known to exist in the folder:
central_storage_basename = TrustBasics.getCentralStorageFilename()
if central_storage_basename not in file_signatures:
self._violation_handler(f"Central storage file not signed for '{path}'.")
return False
# Verify that the central storage file hasn't been tampered with:
if not self._verifyFile(central_storage_filename, file_signatures[central_storage_basename]):
self._violation_handler(f"Central storage file does not match signature for '{path}'.")
return False
# Check if the signing file itself has been tampered with (manifest is self-signed):
if not self._verifyManifestIntegrety(file_signatures, manifest_content):
self._violation_handler(f"Manifest '{manifest_path}' is not properly self-signed in '{path}'.")
return False
# Check if the central storage file doesn't contain files that would be moved outside the plugin folder, or
# files that would be moved to outside of the central storage location:
with open(central_storage_filename, "r", encoding = "utf-8") as central_storage_file:
central_storage_list = json.loads(central_storage_file.read())
storage_location = CentralFileStorage.getCentralStorageLocation()
for file_to_move in central_storage_list:
# Any file is not from outside of the plugin:
source_full_path = os.path.join(path, file_to_move[0])
if not TrustBasics.isPathInLocation(path, source_full_path):
self._violation_handler(f"Item to store '{file_to_move[0]}' is from outside of '{path}'.")
return False
# Any file does not go outside of storage territory:
dest_full_path = os.path.join(storage_location, file_to_move[1])
if not TrustBasics.isPathInLocation(storage_location, dest_full_path):
self._violation_handler(f"Move '{file_to_move[0]}' from '{path}' to outside of storage folder.")
return False
# Otherwise, as far as this quick pre check is concerned, there is nothing wrong:
Logger.log("i", f"Central storage file signed correctly for '{path}'.")
return True
except: # Yes, we do really want this on _every_ exception that might occur.
self._violation_handler(f"Exception during verification of central storage file for '{path}'.")
return False