def prepare(self, reactor, clock, hs):
self.media_repo = hs.get_media_repository_resource()
self.server_name = hs.hostname
self.admin_user = self.register_user("admin", "pass", admin=True)
self.admin_user_tok = self.login("admin", "pass")
self.filepaths = MediaFilePaths(hs.config.media_store_path)
def prepare(self, reactor, clock, hs):
self.handler = hs.get_device_handler()
self.media_repo = hs.get_media_repository_resource()
self.server_name = hs.hostname
self.admin_user = self.register_user("admin", "pass", admin=True)
self.admin_user_tok = self.login("admin", "pass")
self.filepaths = MediaFilePaths(hs.config.media_store_path)
self.url = "/_synapse/admin/v1/media/%s/delete" % self.server_name
def main(src_repo, dest_repo):
src_paths = MediaFilePaths(src_repo)
dest_paths = MediaFilePaths(dest_repo)
for line in sys.stdin:
line = line.strip()
parts = line.split("|")
if len(parts) != 2:
print("Unable to parse input line %s" % line, file=sys.stderr)
exit(1)
move_media(parts[0], parts[1], src_paths, dest_paths)
def prepare(self, reactor, clock, hs):
self.media_repo = hs.get_media_repository_resource()
self.server_name = hs.hostname
self.admin_user = self.register_user("admin", "pass", admin=True)
self.admin_user_tok = self.login("admin", "pass")
self.filepaths = MediaFilePaths(hs.config.media.media_store_path)
self.url = "/_synapse/admin/v1/media/%s/delete" % self.server_name
# Move clock up to somewhat realistic time
self.reactor.advance(1000000000)
def prepare(self, reactor, clock, hs):
self.test_dir = tempfile.mkdtemp(prefix="synapse-tests-")
self.addCleanup(shutil.rmtree, self.test_dir)
self.primary_base_path = os.path.join(self.test_dir, "primary")
self.secondary_base_path = os.path.join(self.test_dir, "secondary")
hs.config.media_store_path = self.primary_base_path
storage_providers = [FileStorageProviderBackend(hs, self.secondary_base_path)]
self.filepaths = MediaFilePaths(self.primary_base_path)
self.media_storage = MediaStorage(
hs, self.primary_base_path, self.filepaths, storage_providers
)
class MediaStorageTests(unittest.HomeserverTestCase):
needs_threadpool = True
def prepare(self, reactor, clock, hs):
self.test_dir = tempfile.mkdtemp(prefix="synapse-tests-")
self.addCleanup(shutil.rmtree, self.test_dir)
self.primary_base_path = os.path.join(self.test_dir, "primary")
self.secondary_base_path = os.path.join(self.test_dir, "secondary")
hs.config.media_store_path = self.primary_base_path
storage_providers = [
FileStorageProviderBackend(hs, self.secondary_base_path)
]
self.filepaths = MediaFilePaths(self.primary_base_path)
self.media_storage = MediaStorage(hs, self.primary_base_path,
self.filepaths, storage_providers)
def test_ensure_media_is_in_local_cache(self):
media_id = "some_media_id"
test_body = "Test\n"
# First we create a file that is in a storage provider but not in the
# local primary media store
rel_path = self.filepaths.local_media_filepath_rel(media_id)
secondary_path = os.path.join(self.secondary_base_path, rel_path)
os.makedirs(os.path.dirname(secondary_path))
with open(secondary_path, "w") as f:
f.write(test_body)
# Now we run ensure_media_is_in_local_cache, which should copy the file
# to the local cache.
file_info = FileInfo(None, media_id)
# This uses a real blocking threadpool so we have to wait for it to be
# actually done :/
x = self.media_storage.ensure_media_is_in_local_cache(file_info)
# Hotloop until the threadpool does its job...
self.wait_on_thread(x)
local_path = self.get_success(x)
self.assertTrue(os.path.exists(local_path))
# Asserts the file is under the expected local cache directory
self.assertEquals(
os.path.commonprefix([self.primary_base_path, local_path]),
self.primary_base_path,
)
with open(local_path) as f:
body = f.read()
self.assertEqual(test_body, body)
def setUp(self):
self.test_dir = tempfile.mkdtemp(prefix="synapse-tests-")
self.primary_base_path = os.path.join(self.test_dir, "primary")
self.secondary_base_path = os.path.join(self.test_dir, "secondary")
hs = Mock()
hs.get_reactor = Mock(return_value=reactor)
hs.config.media_store_path = self.primary_base_path
storage_providers = [FileStorageProviderBackend(hs, self.secondary_base_path)]
self.filepaths = MediaFilePaths(self.primary_base_path)
self.media_storage = MediaStorage(
hs, self.primary_base_path, self.filepaths, storage_providers
)
class MediaStorageTests(unittest.TestCase):
def setUp(self):
self.test_dir = tempfile.mkdtemp(prefix="synapse-tests-")
self.primary_base_path = os.path.join(self.test_dir, "primary")
self.secondary_base_path = os.path.join(self.test_dir, "secondary")
hs = Mock()
hs.get_reactor = Mock(return_value=reactor)
hs.config.media_store_path = self.primary_base_path
storage_providers = [
FileStorageProviderBackend(hs, self.secondary_base_path)
]
self.filepaths = MediaFilePaths(self.primary_base_path)
self.media_storage = MediaStorage(
hs,
self.primary_base_path,
self.filepaths,
storage_providers,
)
def tearDown(self):
shutil.rmtree(self.test_dir)
@defer.inlineCallbacks
def test_ensure_media_is_in_local_cache(self):
media_id = "some_media_id"
test_body = "Test\n"
# First we create a file that is in a storage provider but not in the
# local primary media store
rel_path = self.filepaths.local_media_filepath_rel(media_id)
secondary_path = os.path.join(self.secondary_base_path, rel_path)
os.makedirs(os.path.dirname(secondary_path))
with open(secondary_path, "w") as f:
f.write(test_body)
# Now we run ensure_media_is_in_local_cache, which should copy the file
# to the local cache.
file_info = FileInfo(None, media_id)
local_path = yield self.media_storage.ensure_media_is_in_local_cache(
file_info)
self.assertTrue(os.path.exists(local_path))
# Asserts the file is under the expected local cache directory
self.assertEquals(
os.path.commonprefix([self.primary_base_path, local_path]),
self.primary_base_path,
)
with open(local_path) as f:
body = f.read()
self.assertEqual(test_body, body)
def test_traversal_outside(self) -> None:
"""Test that the jail check fails for paths that escape the media directory."""
filepaths = MediaFilePaths("/media_store")
path = "url_cache/2020-01-02/../../../GerZNDnDZVjsOtar"
with self.assertRaises(ValueError):
self._check_relative_path(filepaths, path)
with self.assertRaises(ValueError):
self._check_absolute_path(filepaths, path)
def test_traversal_inside(self) -> None:
"""Test the jail check for paths that stay within the media directory."""
# Despite the `../`s, these paths still lie within the media directory and it's
# expected for the jail check to allow them through.
# These paths ought to trip the other checks in place and should never be
# returned.
filepaths = MediaFilePaths("/media_store")
path = "url_cache/2020-01-02/../../GerZNDnDZVjsOtar"
self._check_relative_path(filepaths, path)
self._check_absolute_path(filepaths, path)
def prepare(self, reactor: MemoryReactor, clock: Clock,
hs: HomeServer) -> None:
self.media_repo = hs.get_media_repository_resource()
self.server_name = hs.hostname
self.admin_user = self.register_user("admin", "pass", admin=True)
self.admin_user_tok = self.login("admin", "pass")
self.filepaths = MediaFilePaths(hs.config.media.media_store_path)
self.url = "/_synapse/admin/v1/purge_media_cache"
def test_traversal_reentry(self) -> None:
"""Test the jail check for paths that exit and re-enter the media directory."""
# These paths lie outside the media directory if it is a symlink, and inside
# otherwise. Ideally the check should fail, but this proves difficult.
# This test documents the behaviour for this edge case.
# These paths ought to trip the other checks in place and should never be
# returned.
filepaths = MediaFilePaths("/media_store")
path = "url_cache/2020-01-02/../../../media_store/GerZNDnDZVjsOtar"
self._check_relative_path(filepaths, path)
self._check_absolute_path(filepaths, path)
class MediaStorageTests(unittest.TestCase):
def setUp(self):
self.test_dir = tempfile.mkdtemp(prefix="synapse-tests-")
self.primary_base_path = os.path.join(self.test_dir, "primary")
self.secondary_base_path = os.path.join(self.test_dir, "secondary")
hs = Mock()
hs.config.media_store_path = self.primary_base_path
storage_providers = [FileStorageProviderBackend(
hs, self.secondary_base_path
)]
self.filepaths = MediaFilePaths(self.primary_base_path)
self.media_storage = MediaStorage(
self.primary_base_path, self.filepaths, storage_providers,
)
def tearDown(self):
shutil.rmtree(self.test_dir)
@defer.inlineCallbacks
def test_ensure_media_is_in_local_cache(self):
media_id = "some_media_id"
test_body = "Test\n"
# First we create a file that is in a storage provider but not in the
# local primary media store
rel_path = self.filepaths.local_media_filepath_rel(media_id)
secondary_path = os.path.join(self.secondary_base_path, rel_path)
os.makedirs(os.path.dirname(secondary_path))
with open(secondary_path, "w") as f:
f.write(test_body)
# Now we run ensure_media_is_in_local_cache, which should copy the file
# to the local cache.
file_info = FileInfo(None, media_id)
local_path = yield self.media_storage.ensure_media_is_in_local_cache(file_info)
self.assertTrue(os.path.exists(local_path))
# Asserts the file is under the expected local cache directory
self.assertEquals(
os.path.commonprefix([self.primary_base_path, local_path]),
self.primary_base_path,
)
with open(local_path) as f:
body = f.read()
self.assertEqual(test_body, body)
def test_symlink(self) -> None:
"""Test that a symlink does not cause the jail check to fail."""
media_store_path = self.mktemp()
# symlink the media store directory
os.symlink("/mnt/synapse/media_store", media_store_path)
# Test that relative and absolute paths don't trip the check
# NB: `media_store_path` is a relative path
filepaths = MediaFilePaths(media_store_path)
self._check_relative_path(filepaths,
"url_cache/2020-01-02/GerZNDnDZVjsOtar")
self._check_absolute_path(filepaths,
"url_cache/2020-01-02/GerZNDnDZVjsOtar")
filepaths = MediaFilePaths(os.path.abspath(media_store_path))
self._check_relative_path(filepaths,
"url_cache/2020-01-02/GerZNDnDZVjsOtar")
self._check_absolute_path(filepaths,
"url_cache/2020-01-02/GerZNDnDZVjsOtar")
def move_media(
origin_server: str,
file_id: str,
src_paths: MediaFilePaths,
dest_paths: MediaFilePaths,
) -> None:
"""Move the given file, and any thumbnails, to the dest repo
Args:
origin_server:
file_id:
src_paths:
dest_paths:
"""
logger.info("%s/%s", origin_server, file_id)
# check that the original exists
original_file = src_paths.remote_media_filepath(origin_server, file_id)
if not os.path.exists(original_file):
logger.warning(
"Original for %s/%s (%s) does not exist",
origin_server,
file_id,
original_file,
)
else:
mkdir_and_move(
original_file,
dest_paths.remote_media_filepath(origin_server, file_id))
# now look for thumbnails
original_thumb_dir = src_paths.remote_media_thumbnail_dir(
origin_server, file_id)
if not os.path.exists(original_thumb_dir):
return
mkdir_and_move(
original_thumb_dir,
dest_paths.remote_media_thumbnail_dir(origin_server, file_id),
)
class MediaFilePathsTestCase(unittest.TestCase):
def setUp(self):
super().setUp()
self.filepaths = MediaFilePaths("/media_store")
def test_local_media_filepath(self):
"""Test local media paths"""
self.assertEqual(
self.filepaths.local_media_filepath_rel("GerZNDnDZVjsOtardLuwfIBg"),
"local_content/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
self.assertEqual(
self.filepaths.local_media_filepath("GerZNDnDZVjsOtardLuwfIBg"),
"/media_store/local_content/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
def test_local_media_thumbnail(self):
"""Test local media thumbnail paths"""
self.assertEqual(
self.filepaths.local_media_thumbnail_rel(
"GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale"
),
"local_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale",
)
self.assertEqual(
self.filepaths.local_media_thumbnail(
"GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale"
),
"/media_store/local_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale",
)
def test_local_media_thumbnail_dir(self):
"""Test local media thumbnail directory paths"""
self.assertEqual(
self.filepaths.local_media_thumbnail_dir("GerZNDnDZVjsOtardLuwfIBg"),
"/media_store/local_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
def test_remote_media_filepath(self):
"""Test remote media paths"""
self.assertEqual(
self.filepaths.remote_media_filepath_rel(
"example.com", "GerZNDnDZVjsOtardLuwfIBg"
),
"remote_content/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
self.assertEqual(
self.filepaths.remote_media_filepath(
"example.com", "GerZNDnDZVjsOtardLuwfIBg"
),
"/media_store/remote_content/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
def test_remote_media_thumbnail(self):
"""Test remote media thumbnail paths"""
self.assertEqual(
self.filepaths.remote_media_thumbnail_rel(
"example.com",
"GerZNDnDZVjsOtardLuwfIBg",
800,
600,
"image/jpeg",
"scale",
),
"remote_thumbnail/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale",
)
self.assertEqual(
self.filepaths.remote_media_thumbnail(
"example.com",
"GerZNDnDZVjsOtardLuwfIBg",
800,
600,
"image/jpeg",
"scale",
),
"/media_store/remote_thumbnail/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale",
)
def test_remote_media_thumbnail_legacy(self):
"""Test old-style remote media thumbnail paths"""
self.assertEqual(
self.filepaths.remote_media_thumbnail_rel_legacy(
"example.com", "GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg"
),
"remote_thumbnail/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg",
)
def test_remote_media_thumbnail_dir(self):
"""Test remote media thumbnail directory paths"""
self.assertEqual(
self.filepaths.remote_media_thumbnail_dir(
"example.com", "GerZNDnDZVjsOtardLuwfIBg"
),
"/media_store/remote_thumbnail/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
def test_url_cache_filepath(self):
"""Test URL cache paths"""
self.assertEqual(
self.filepaths.url_cache_filepath_rel("2020-01-02_GerZNDnDZVjsOtar"),
"url_cache/2020-01-02/GerZNDnDZVjsOtar",
)
self.assertEqual(
self.filepaths.url_cache_filepath("2020-01-02_GerZNDnDZVjsOtar"),
"/media_store/url_cache/2020-01-02/GerZNDnDZVjsOtar",
)
def test_url_cache_filepath_legacy(self):
"""Test old-style URL cache paths"""
self.assertEqual(
self.filepaths.url_cache_filepath_rel("GerZNDnDZVjsOtardLuwfIBg"),
"url_cache/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
self.assertEqual(
self.filepaths.url_cache_filepath("GerZNDnDZVjsOtardLuwfIBg"),
"/media_store/url_cache/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
def test_url_cache_filepath_dirs_to_delete(self):
"""Test URL cache cleanup paths"""
self.assertEqual(
self.filepaths.url_cache_filepath_dirs_to_delete(
"2020-01-02_GerZNDnDZVjsOtar"
),
["/media_store/url_cache/2020-01-02"],
)
def test_url_cache_filepath_dirs_to_delete_legacy(self):
"""Test old-style URL cache cleanup paths"""
self.assertEqual(
self.filepaths.url_cache_filepath_dirs_to_delete(
"GerZNDnDZVjsOtardLuwfIBg"
),
[
"/media_store/url_cache/Ge/rZ",
"/media_store/url_cache/Ge",
],
)
def test_url_cache_thumbnail(self):
"""Test URL cache thumbnail paths"""
self.assertEqual(
self.filepaths.url_cache_thumbnail_rel(
"2020-01-02_GerZNDnDZVjsOtar", 800, 600, "image/jpeg", "scale"
),
"url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar/800-600-image-jpeg-scale",
)
self.assertEqual(
self.filepaths.url_cache_thumbnail(
"2020-01-02_GerZNDnDZVjsOtar", 800, 600, "image/jpeg", "scale"
),
"/media_store/url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar/800-600-image-jpeg-scale",
)
def test_url_cache_thumbnail_legacy(self):
"""Test old-style URL cache thumbnail paths"""
self.assertEqual(
self.filepaths.url_cache_thumbnail_rel(
"GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale"
),
"url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale",
)
self.assertEqual(
self.filepaths.url_cache_thumbnail(
"GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale"
),
"/media_store/url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale",
)
def test_url_cache_thumbnail_directory(self):
"""Test URL cache thumbnail directory paths"""
self.assertEqual(
self.filepaths.url_cache_thumbnail_directory_rel(
"2020-01-02_GerZNDnDZVjsOtar"
),
"url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar",
)
self.assertEqual(
self.filepaths.url_cache_thumbnail_directory("2020-01-02_GerZNDnDZVjsOtar"),
"/media_store/url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar",
)
def test_url_cache_thumbnail_directory_legacy(self):
"""Test old-style URL cache thumbnail directory paths"""
self.assertEqual(
self.filepaths.url_cache_thumbnail_directory_rel(
"GerZNDnDZVjsOtardLuwfIBg"
),
"url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
self.assertEqual(
self.filepaths.url_cache_thumbnail_directory("GerZNDnDZVjsOtardLuwfIBg"),
"/media_store/url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
def test_url_cache_thumbnail_dirs_to_delete(self):
"""Test URL cache thumbnail cleanup paths"""
self.assertEqual(
self.filepaths.url_cache_thumbnail_dirs_to_delete(
"2020-01-02_GerZNDnDZVjsOtar"
),
[
"/media_store/url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar",
"/media_store/url_cache_thumbnails/2020-01-02",
],
)
def test_url_cache_thumbnail_dirs_to_delete_legacy(self):
"""Test old-style URL cache thumbnail cleanup paths"""
self.assertEqual(
self.filepaths.url_cache_thumbnail_dirs_to_delete(
"GerZNDnDZVjsOtardLuwfIBg"
),
[
"/media_store/url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg",
"/media_store/url_cache_thumbnails/Ge/rZ",
"/media_store/url_cache_thumbnails/Ge",
],
)
class DeleteMediaByIDTestCase(unittest.HomeserverTestCase):
servlets = [
synapse.rest.admin.register_servlets,
synapse.rest.admin.register_servlets_for_media_repo,
login.register_servlets,
]
def prepare(self, reactor: MemoryReactor, clock: Clock,
hs: HomeServer) -> None:
self.media_repo = hs.get_media_repository_resource()
self.server_name = hs.hostname
self.admin_user = self.register_user("admin", "pass", admin=True)
self.admin_user_tok = self.login("admin", "pass")
self.filepaths = MediaFilePaths(hs.config.media.media_store_path)
def test_no_auth(self) -> None:
"""
Try to delete media without authentication.
"""
url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345")
channel = self.make_request("DELETE", url, b"{}")
self.assertEqual(
HTTPStatus.UNAUTHORIZED,
channel.code,
msg=channel.json_body,
)
self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
def test_requester_is_no_admin(self) -> None:
"""
If the user is not a server admin, an error is returned.
"""
self.other_user = self.register_user("user", "pass")
self.other_user_token = self.login("user", "pass")
url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345")
channel = self.make_request(
"DELETE",
url,
access_token=self.other_user_token,
)
self.assertEqual(
HTTPStatus.FORBIDDEN,
channel.code,
msg=channel.json_body,
)
self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
def test_media_does_not_exist(self) -> None:
"""
Tests that a lookup for a media that does not exist returns a HTTPStatus.NOT_FOUND
"""
url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345")
channel = self.make_request(
"DELETE",
url,
access_token=self.admin_user_tok,
)
self.assertEqual(HTTPStatus.NOT_FOUND,
channel.code,
msg=channel.json_body)
self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"])
def test_media_is_not_local(self) -> None:
"""
Tests that a lookup for a media that is not a local returns a HTTPStatus.BAD_REQUEST
"""
url = "/_synapse/admin/v1/media/%s/%s" % ("unknown_domain", "12345")
channel = self.make_request(
"DELETE",
url,
access_token=self.admin_user_tok,
)
self.assertEqual(HTTPStatus.BAD_REQUEST,
channel.code,
msg=channel.json_body)
self.assertEqual("Can only delete local media",
channel.json_body["error"])
def test_delete_media(self) -> None:
"""
Tests that delete a media is successfully
"""
download_resource = self.media_repo.children[b"download"]
upload_resource = self.media_repo.children[b"upload"]
# Upload some media into the room
response = self.helper.upload_media(
upload_resource,
SMALL_PNG,
tok=self.admin_user_tok,
expect_code=HTTPStatus.OK,
)
# Extract media ID from the response
server_and_media_id = response["content_uri"][6:] # Cut off 'mxc://'
server_name, media_id = server_and_media_id.split("/")
self.assertEqual(server_name, self.server_name)
# Attempt to access media
channel = make_request(
self.reactor,
FakeSite(download_resource, self.reactor),
"GET",
server_and_media_id,
shorthand=False,
access_token=self.admin_user_tok,
)
# Should be successful
self.assertEqual(
HTTPStatus.OK,
channel.code,
msg=("Expected to receive a HTTPStatus.OK on accessing media: %s" %
server_and_media_id),
)
# Test if the file exists
local_path = self.filepaths.local_media_filepath(media_id)
self.assertTrue(os.path.exists(local_path))
url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, media_id)
# Delete media
channel = self.make_request(
"DELETE",
url,
access_token=self.admin_user_tok,
)
self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
media_id,
channel.json_body["deleted_media"][0],
)
# Attempt to access media
channel = make_request(
self.reactor,
FakeSite(download_resource, self.reactor),
"GET",
server_and_media_id,
shorthand=False,
access_token=self.admin_user_tok,
)
self.assertEqual(
HTTPStatus.NOT_FOUND,
channel.code,
msg=
("Expected to receive a HTTPStatus.NOT_FOUND on accessing deleted media: %s"
% server_and_media_id),
)
# Test if the file is deleted
self.assertFalse(os.path.exists(local_path))
class MediaFilePathsTestCase(unittest.TestCase):
def setUp(self):
super().setUp()
self.filepaths = MediaFilePaths("/media_store")
def test_local_media_filepath(self):
"""Test local media paths"""
self.assertEqual(
self.filepaths.local_media_filepath_rel(
"GerZNDnDZVjsOtardLuwfIBg"),
"local_content/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
self.assertEqual(
self.filepaths.local_media_filepath("GerZNDnDZVjsOtardLuwfIBg"),
"/media_store/local_content/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
def test_local_media_thumbnail(self):
"""Test local media thumbnail paths"""
self.assertEqual(
self.filepaths.local_media_thumbnail_rel(
"GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale"),
"local_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale",
)
self.assertEqual(
self.filepaths.local_media_thumbnail("GerZNDnDZVjsOtardLuwfIBg",
800, 600, "image/jpeg",
"scale"),
"/media_store/local_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale",
)
def test_local_media_thumbnail_dir(self):
"""Test local media thumbnail directory paths"""
self.assertEqual(
self.filepaths.local_media_thumbnail_dir(
"GerZNDnDZVjsOtardLuwfIBg"),
"/media_store/local_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
def test_remote_media_filepath(self):
"""Test remote media paths"""
self.assertEqual(
self.filepaths.remote_media_filepath_rel(
"example.com", "GerZNDnDZVjsOtardLuwfIBg"),
"remote_content/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
self.assertEqual(
self.filepaths.remote_media_filepath("example.com",
"GerZNDnDZVjsOtardLuwfIBg"),
"/media_store/remote_content/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
def test_remote_media_thumbnail(self):
"""Test remote media thumbnail paths"""
self.assertEqual(
self.filepaths.remote_media_thumbnail_rel(
"example.com",
"GerZNDnDZVjsOtardLuwfIBg",
800,
600,
"image/jpeg",
"scale",
),
"remote_thumbnail/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale",
)
self.assertEqual(
self.filepaths.remote_media_thumbnail(
"example.com",
"GerZNDnDZVjsOtardLuwfIBg",
800,
600,
"image/jpeg",
"scale",
),
"/media_store/remote_thumbnail/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale",
)
def test_remote_media_thumbnail_legacy(self):
"""Test old-style remote media thumbnail paths"""
self.assertEqual(
self.filepaths.remote_media_thumbnail_rel_legacy(
"example.com", "GerZNDnDZVjsOtardLuwfIBg", 800, 600,
"image/jpeg"),
"remote_thumbnail/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg",
)
def test_remote_media_thumbnail_dir(self):
"""Test remote media thumbnail directory paths"""
self.assertEqual(
self.filepaths.remote_media_thumbnail_dir(
"example.com", "GerZNDnDZVjsOtardLuwfIBg"),
"/media_store/remote_thumbnail/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
def test_url_cache_filepath(self):
"""Test URL cache paths"""
self.assertEqual(
self.filepaths.url_cache_filepath_rel(
"2020-01-02_GerZNDnDZVjsOtar"),
"url_cache/2020-01-02/GerZNDnDZVjsOtar",
)
self.assertEqual(
self.filepaths.url_cache_filepath("2020-01-02_GerZNDnDZVjsOtar"),
"/media_store/url_cache/2020-01-02/GerZNDnDZVjsOtar",
)
def test_url_cache_filepath_legacy(self):
"""Test old-style URL cache paths"""
self.assertEqual(
self.filepaths.url_cache_filepath_rel("GerZNDnDZVjsOtardLuwfIBg"),
"url_cache/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
self.assertEqual(
self.filepaths.url_cache_filepath("GerZNDnDZVjsOtardLuwfIBg"),
"/media_store/url_cache/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
def test_url_cache_filepath_dirs_to_delete(self):
"""Test URL cache cleanup paths"""
self.assertEqual(
self.filepaths.url_cache_filepath_dirs_to_delete(
"2020-01-02_GerZNDnDZVjsOtar"),
["/media_store/url_cache/2020-01-02"],
)
def test_url_cache_filepath_dirs_to_delete_legacy(self):
"""Test old-style URL cache cleanup paths"""
self.assertEqual(
self.filepaths.url_cache_filepath_dirs_to_delete(
"GerZNDnDZVjsOtardLuwfIBg"),
[
"/media_store/url_cache/Ge/rZ",
"/media_store/url_cache/Ge",
],
)
def test_url_cache_thumbnail(self):
"""Test URL cache thumbnail paths"""
self.assertEqual(
self.filepaths.url_cache_thumbnail_rel(
"2020-01-02_GerZNDnDZVjsOtar", 800, 600, "image/jpeg",
"scale"),
"url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar/800-600-image-jpeg-scale",
)
self.assertEqual(
self.filepaths.url_cache_thumbnail("2020-01-02_GerZNDnDZVjsOtar",
800, 600, "image/jpeg",
"scale"),
"/media_store/url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar/800-600-image-jpeg-scale",
)
def test_url_cache_thumbnail_legacy(self):
"""Test old-style URL cache thumbnail paths"""
self.assertEqual(
self.filepaths.url_cache_thumbnail_rel("GerZNDnDZVjsOtardLuwfIBg",
800, 600, "image/jpeg",
"scale"),
"url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale",
)
self.assertEqual(
self.filepaths.url_cache_thumbnail("GerZNDnDZVjsOtardLuwfIBg", 800,
600, "image/jpeg", "scale"),
"/media_store/url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale",
)
def test_url_cache_thumbnail_directory(self):
"""Test URL cache thumbnail directory paths"""
self.assertEqual(
self.filepaths.url_cache_thumbnail_directory_rel(
"2020-01-02_GerZNDnDZVjsOtar"),
"url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar",
)
self.assertEqual(
self.filepaths.url_cache_thumbnail_directory(
"2020-01-02_GerZNDnDZVjsOtar"),
"/media_store/url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar",
)
def test_url_cache_thumbnail_directory_legacy(self):
"""Test old-style URL cache thumbnail directory paths"""
self.assertEqual(
self.filepaths.url_cache_thumbnail_directory_rel(
"GerZNDnDZVjsOtardLuwfIBg"),
"url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
self.assertEqual(
self.filepaths.url_cache_thumbnail_directory(
"GerZNDnDZVjsOtardLuwfIBg"),
"/media_store/url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg",
)
def test_url_cache_thumbnail_dirs_to_delete(self):
"""Test URL cache thumbnail cleanup paths"""
self.assertEqual(
self.filepaths.url_cache_thumbnail_dirs_to_delete(
"2020-01-02_GerZNDnDZVjsOtar"),
[
"/media_store/url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar",
"/media_store/url_cache_thumbnails/2020-01-02",
],
)
def test_url_cache_thumbnail_dirs_to_delete_legacy(self):
"""Test old-style URL cache thumbnail cleanup paths"""
self.assertEqual(
self.filepaths.url_cache_thumbnail_dirs_to_delete(
"GerZNDnDZVjsOtardLuwfIBg"),
[
"/media_store/url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg",
"/media_store/url_cache_thumbnails/Ge/rZ",
"/media_store/url_cache_thumbnails/Ge",
],
)
def test_server_name_validation(self):
"""Test validation of server names"""
self._test_path_validation(
[
"remote_media_filepath_rel",
"remote_media_filepath",
"remote_media_thumbnail_rel",
"remote_media_thumbnail",
"remote_media_thumbnail_rel_legacy",
"remote_media_thumbnail_dir",
],
parameter="server_name",
valid_values=[
"matrix.org",
"matrix.org:8448",
"matrix-federation.matrix.org",
"matrix-federation.matrix.org:8448",
"10.1.12.123",
"10.1.12.123:8448",
"[fd00:abcd::ffff]",
"[fd00:abcd::ffff]:8448",
],
invalid_values=[
"/matrix.org",
"matrix.org/..",
"matrix.org\x00",
"",
".",
"..",
"/",
],
)
def test_file_id_validation(self):
"""Test validation of local, remote and legacy URL cache file / media IDs"""
# File / media IDs get split into three parts to form paths, consisting of the
# first two characters, next two characters and rest of the ID.
valid_file_ids = [
"GerZNDnDZVjsOtardLuwfIBg",
# Unexpected, but produces an acceptable path:
"GerZN", # "N" becomes the last directory
]
invalid_file_ids = [
"/erZNDnDZVjsOtardLuwfIBg",
"Ge/ZNDnDZVjsOtardLuwfIBg",
"GerZ/DnDZVjsOtardLuwfIBg",
"GerZ/..",
"G\x00rZNDnDZVjsOtardLuwfIBg",
"Ger\x00NDnDZVjsOtardLuwfIBg",
"GerZNDnDZVjsOtardLuwfIBg\x00",
"",
"Ge",
"GerZ",
"GerZ.",
"..rZNDnDZVjsOtardLuwfIBg",
"Ge..NDnDZVjsOtardLuwfIBg",
"GerZ..",
"GerZ/",
]
self._test_path_validation(
[
"local_media_filepath_rel",
"local_media_filepath",
"local_media_thumbnail_rel",
"local_media_thumbnail",
"local_media_thumbnail_dir",
# Legacy URL cache media IDs
"url_cache_filepath_rel",
"url_cache_filepath",
# `url_cache_filepath_dirs_to_delete` is tested below.
"url_cache_thumbnail_rel",
"url_cache_thumbnail",
"url_cache_thumbnail_directory_rel",
"url_cache_thumbnail_directory",
"url_cache_thumbnail_dirs_to_delete",
],
parameter="media_id",
valid_values=valid_file_ids,
invalid_values=invalid_file_ids,
)
# `url_cache_filepath_dirs_to_delete` ignores what would be the last path
# component, so only the first 4 characters matter.
self._test_path_validation(
[
"url_cache_filepath_dirs_to_delete",
],
parameter="media_id",
valid_values=valid_file_ids,
invalid_values=[
"/erZNDnDZVjsOtardLuwfIBg",
"Ge/ZNDnDZVjsOtardLuwfIBg",
"G\x00rZNDnDZVjsOtardLuwfIBg",
"Ger\x00NDnDZVjsOtardLuwfIBg",
"",
"Ge",
"..rZNDnDZVjsOtardLuwfIBg",
"Ge..NDnDZVjsOtardLuwfIBg",
],
)
self._test_path_validation(
[
"remote_media_filepath_rel",
"remote_media_filepath",
"remote_media_thumbnail_rel",
"remote_media_thumbnail",
"remote_media_thumbnail_rel_legacy",
"remote_media_thumbnail_dir",
],
parameter="file_id",
valid_values=valid_file_ids,
invalid_values=invalid_file_ids,
)
def test_url_cache_media_id_validation(self):
"""Test validation of URL cache media IDs"""
self._test_path_validation(
[
"url_cache_filepath_rel",
"url_cache_filepath",
# `url_cache_filepath_dirs_to_delete` only cares about the date prefix
"url_cache_thumbnail_rel",
"url_cache_thumbnail",
"url_cache_thumbnail_directory_rel",
"url_cache_thumbnail_directory",
"url_cache_thumbnail_dirs_to_delete",
],
parameter="media_id",
valid_values=[
"2020-01-02_GerZNDnDZVjsOtar",
"2020-01-02_G", # Unexpected, but produces an acceptable path
],
invalid_values=[
"2020-01-02",
"2020-01-02-",
"2020-01-02-.",
"2020-01-02-..",
"2020-01-02-/",
"2020-01-02-/GerZNDnDZVjsOtar",
"2020-01-02-GerZNDnDZVjsOtar/..",
"2020-01-02-GerZNDnDZVjsOtar\x00",
],
)
def test_content_type_validation(self):
"""Test validation of thumbnail content types"""
self._test_path_validation(
[
"local_media_thumbnail_rel",
"local_media_thumbnail",
"remote_media_thumbnail_rel",
"remote_media_thumbnail",
"remote_media_thumbnail_rel_legacy",
"url_cache_thumbnail_rel",
"url_cache_thumbnail",
],
parameter="content_type",
valid_values=[
"image/jpeg",
],
invalid_values=[
"", # ValueError: not enough values to unpack
"image/jpeg/abc", # ValueError: too many values to unpack
"image/jpeg\x00",
],
)
def test_thumbnail_method_validation(self):
"""Test validation of thumbnail methods"""
self._test_path_validation(
[
"local_media_thumbnail_rel",
"local_media_thumbnail",
"remote_media_thumbnail_rel",
"remote_media_thumbnail",
"url_cache_thumbnail_rel",
"url_cache_thumbnail",
],
parameter="method",
valid_values=[
"crop",
"scale",
],
invalid_values=[
"/scale",
"scale/..",
"scale\x00",
"/",
],
)
def _test_path_validation(
self,
methods: Iterable[str],
parameter: str,
valid_values: Iterable[str],
invalid_values: Iterable[str],
):
"""Test that the specified methods validate the named parameter as expected
Args:
methods: The names of `MediaFilePaths` methods to test
parameter: The name of the parameter to test
valid_values: A list of parameter values that are expected to be accepted
invalid_values: A list of parameter values that are expected to be rejected
Raises:
AssertionError: If a value was accepted when it should have failed
validation.
ValueError: If a value failed validation when it should have been accepted.
"""
for method in methods:
get_path = getattr(self.filepaths, method)
parameters = inspect.signature(get_path).parameters
kwargs = {
"server_name": "matrix.org",
"media_id": "GerZNDnDZVjsOtardLuwfIBg",
"file_id": "GerZNDnDZVjsOtardLuwfIBg",
"width": 800,
"height": 600,
"content_type": "image/jpeg",
"method": "scale",
}
if get_path.__name__.startswith("url_"):
kwargs["media_id"] = "2020-01-02_GerZNDnDZVjsOtar"
kwargs = {k: v for k, v in kwargs.items() if k in parameters}
kwargs.pop(parameter)
for value in valid_values:
kwargs[parameter] = value
get_path(**kwargs)
# No exception should be raised
for value in invalid_values:
with self.assertRaises(ValueError):
kwargs[parameter] = value
path_or_list = get_path(**kwargs)
self.fail(f"{value!r} unexpectedly passed validation: "
f"{method} returned {path_or_list!r}")
def setUp(self):
super().setUp()
self.filepaths = MediaFilePaths("/media_store")
class DeleteMediaByIDTestCase(unittest.HomeserverTestCase):
servlets = [
synapse.rest.admin.register_servlets,
synapse.rest.admin.register_servlets_for_media_repo,
login.register_servlets,
]
def prepare(self, reactor, clock, hs):
self.media_repo = hs.get_media_repository_resource()
self.server_name = hs.hostname
self.admin_user = self.register_user("admin", "pass", admin=True)
self.admin_user_tok = self.login("admin", "pass")
self.filepaths = MediaFilePaths(hs.config.media_store_path)
def test_no_auth(self):
"""
Try to delete media without authentication.
"""
url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345")
channel = self.make_request("DELETE", url, b"{}")
self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
def test_requester_is_no_admin(self):
"""
If the user is not a server admin, an error is returned.
"""
self.other_user = self.register_user("user", "pass")
self.other_user_token = self.login("user", "pass")
url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345")
channel = self.make_request(
"DELETE",
url,
access_token=self.other_user_token,
)
self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
def test_media_does_not_exist(self):
"""
Tests that a lookup for a media that does not exist returns a 404
"""
url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345")
channel = self.make_request(
"DELETE",
url,
access_token=self.admin_user_tok,
)
self.assertEqual(404, channel.code, msg=channel.json_body)
self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"])
def test_media_is_not_local(self):
"""
Tests that a lookup for a media that is not a local returns a 400
"""
url = "/_synapse/admin/v1/media/%s/%s" % ("unknown_domain", "12345")
channel = self.make_request(
"DELETE",
url,
access_token=self.admin_user_tok,
)
self.assertEqual(400, channel.code, msg=channel.json_body)
self.assertEqual("Can only delete local media", channel.json_body["error"])
def test_delete_media(self):
"""
Tests that delete a media is successfully
"""
download_resource = self.media_repo.children[b"download"]
upload_resource = self.media_repo.children[b"upload"]
image_data = unhexlify(
b"89504e470d0a1a0a0000000d4948445200000001000000010806"
b"0000001f15c4890000000a49444154789c63000100000500010d"
b"0a2db40000000049454e44ae426082"
)
# Upload some media into the room
response = self.helper.upload_media(
upload_resource, image_data, tok=self.admin_user_tok, expect_code=200
)
# Extract media ID from the response
server_and_media_id = response["content_uri"][6:] # Cut off 'mxc://'
server_name, media_id = server_and_media_id.split("/")
self.assertEqual(server_name, self.server_name)
# Attempt to access media
channel = make_request(
self.reactor,
FakeSite(download_resource),
"GET",
server_and_media_id,
shorthand=False,
access_token=self.admin_user_tok,
)
# Should be successful
self.assertEqual(
200,
channel.code,
msg=(
"Expected to receive a 200 on accessing media: %s" % server_and_media_id
),
)
# Test if the file exists
local_path = self.filepaths.local_media_filepath(media_id)
self.assertTrue(os.path.exists(local_path))
url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, media_id)
# Delete media
channel = self.make_request(
"DELETE",
url,
access_token=self.admin_user_tok,
)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
media_id,
channel.json_body["deleted_media"][0],
)
# Attempt to access media
channel = make_request(
self.reactor,
FakeSite(download_resource),
"GET",
server_and_media_id,
shorthand=False,
access_token=self.admin_user_tok,
)
self.assertEqual(
404,
channel.code,
msg=(
"Expected to receive a 404 on accessing deleted media: %s"
% server_and_media_id
),
)
# Test if the file is deleted
self.assertFalse(os.path.exists(local_path))
class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase):
servlets = [
synapse.rest.admin.register_servlets,
synapse.rest.admin.register_servlets_for_media_repo,
login.register_servlets,
profile.register_servlets,
room.register_servlets,
]
def prepare(self, reactor, clock, hs):
self.media_repo = hs.get_media_repository_resource()
self.server_name = hs.hostname
self.admin_user = self.register_user("admin", "pass", admin=True)
self.admin_user_tok = self.login("admin", "pass")
self.filepaths = MediaFilePaths(hs.config.media_store_path)
self.url = "/_synapse/admin/v1/media/%s/delete" % self.server_name
def test_no_auth(self):
"""
Try to delete media without authentication.
"""
channel = self.make_request("POST", self.url, b"{}")
self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
def test_requester_is_no_admin(self):
"""
If the user is not a server admin, an error is returned.
"""
self.other_user = self.register_user("user", "pass")
self.other_user_token = self.login("user", "pass")
channel = self.make_request(
"POST",
self.url,
access_token=self.other_user_token,
)
self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
def test_media_is_not_local(self):
"""
Tests that a lookup for media that is not local returns a 400
"""
url = "/_synapse/admin/v1/media/%s/delete" % "unknown_domain"
channel = self.make_request(
"POST",
url + "?before_ts=1234",
access_token=self.admin_user_tok,
)
self.assertEqual(400, channel.code, msg=channel.json_body)
self.assertEqual("Can only delete local media", channel.json_body["error"])
def test_missing_parameter(self):
"""
If the parameter `before_ts` is missing, an error is returned.
"""
channel = self.make_request(
"POST",
self.url,
access_token=self.admin_user_tok,
)
self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.MISSING_PARAM, channel.json_body["errcode"])
self.assertEqual(
"Missing integer query parameter b'before_ts'", channel.json_body["error"]
)
def test_invalid_parameter(self):
"""
If parameters are invalid, an error is returned.
"""
channel = self.make_request(
"POST",
self.url + "?before_ts=-1234",
access_token=self.admin_user_tok,
)
self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
self.assertEqual(
"Query parameter before_ts must be a string representing a positive integer.",
channel.json_body["error"],
)
channel = self.make_request(
"POST",
self.url + "?before_ts=1234&size_gt=-1234",
access_token=self.admin_user_tok,
)
self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
self.assertEqual(
"Query parameter size_gt must be a string representing a positive integer.",
channel.json_body["error"],
)
channel = self.make_request(
"POST",
self.url + "?before_ts=1234&keep_profiles=not_bool",
access_token=self.admin_user_tok,
)
self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.UNKNOWN, channel.json_body["errcode"])
self.assertEqual(
"Boolean query parameter b'keep_profiles' must be one of ['true', 'false']",
channel.json_body["error"],
)
def test_delete_media_never_accessed(self):
"""
Tests that media deleted if it is older than `before_ts` and never accessed
`last_access_ts` is `NULL` and `created_ts` < `before_ts`
"""
# upload and do not access
server_and_media_id = self._create_media()
self.pump(1.0)
# test that the file exists
media_id = server_and_media_id.split("/")[1]
local_path = self.filepaths.local_media_filepath(media_id)
self.assertTrue(os.path.exists(local_path))
# timestamp after upload/create
now_ms = self.clock.time_msec()
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms),
access_token=self.admin_user_tok,
)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
media_id,
channel.json_body["deleted_media"][0],
)
self._access_media(server_and_media_id, False)
def test_keep_media_by_date(self):
"""
Tests that media is not deleted if it is newer than `before_ts`
"""
# timestamp before upload
now_ms = self.clock.time_msec()
server_and_media_id = self._create_media()
self._access_media(server_and_media_id)
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms),
access_token=self.admin_user_tok,
)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(0, channel.json_body["total"])
self._access_media(server_and_media_id)
# timestamp after upload
now_ms = self.clock.time_msec()
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms),
access_token=self.admin_user_tok,
)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
server_and_media_id.split("/")[1],
channel.json_body["deleted_media"][0],
)
self._access_media(server_and_media_id, False)
def test_keep_media_by_size(self):
"""
Tests that media is not deleted if its size is smaller than or equal
to `size_gt`
"""
server_and_media_id = self._create_media()
self._access_media(server_and_media_id)
now_ms = self.clock.time_msec()
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&size_gt=67",
access_token=self.admin_user_tok,
)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(0, channel.json_body["total"])
self._access_media(server_and_media_id)
now_ms = self.clock.time_msec()
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&size_gt=66",
access_token=self.admin_user_tok,
)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
server_and_media_id.split("/")[1],
channel.json_body["deleted_media"][0],
)
self._access_media(server_and_media_id, False)
def test_keep_media_by_user_avatar(self):
"""
Tests that we do not delete media if is used as a user avatar
Tests parameter `keep_profiles`
"""
server_and_media_id = self._create_media()
self._access_media(server_and_media_id)
# set media as avatar
channel = self.make_request(
"PUT",
"/profile/%s/avatar_url" % (self.admin_user,),
content=json.dumps({"avatar_url": "mxc://%s" % (server_and_media_id,)}),
access_token=self.admin_user_tok,
)
self.assertEqual(200, channel.code, msg=channel.json_body)
now_ms = self.clock.time_msec()
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=true",
access_token=self.admin_user_tok,
)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(0, channel.json_body["total"])
self._access_media(server_and_media_id)
now_ms = self.clock.time_msec()
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=false",
access_token=self.admin_user_tok,
)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
server_and_media_id.split("/")[1],
channel.json_body["deleted_media"][0],
)
self._access_media(server_and_media_id, False)
def test_keep_media_by_room_avatar(self):
"""
Tests that we do not delete media if it is used as a room avatar
Tests parameter `keep_profiles`
"""
server_and_media_id = self._create_media()
self._access_media(server_and_media_id)
# set media as room avatar
room_id = self.helper.create_room_as(self.admin_user, tok=self.admin_user_tok)
channel = self.make_request(
"PUT",
"/rooms/%s/state/m.room.avatar" % (room_id,),
content=json.dumps({"url": "mxc://%s" % (server_and_media_id,)}),
access_token=self.admin_user_tok,
)
self.assertEqual(200, channel.code, msg=channel.json_body)
now_ms = self.clock.time_msec()
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=true",
access_token=self.admin_user_tok,
)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(0, channel.json_body["total"])
self._access_media(server_and_media_id)
now_ms = self.clock.time_msec()
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=false",
access_token=self.admin_user_tok,
)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
server_and_media_id.split("/")[1],
channel.json_body["deleted_media"][0],
)
self._access_media(server_and_media_id, False)
def _create_media(self):
"""
Create a media and return media_id and server_and_media_id
"""
upload_resource = self.media_repo.children[b"upload"]
# file size is 67 Byte
image_data = unhexlify(
b"89504e470d0a1a0a0000000d4948445200000001000000010806"
b"0000001f15c4890000000a49444154789c63000100000500010d"
b"0a2db40000000049454e44ae426082"
)
# Upload some media into the room
response = self.helper.upload_media(
upload_resource, image_data, tok=self.admin_user_tok, expect_code=200
)
# Extract media ID from the response
server_and_media_id = response["content_uri"][6:] # Cut off 'mxc://'
server_name = server_and_media_id.split("/")[0]
# Check that new media is a local and not remote
self.assertEqual(server_name, self.server_name)
return server_and_media_id
def _access_media(self, server_and_media_id, expect_success=True):
"""
Try to access a media and check the result
"""
download_resource = self.media_repo.children[b"download"]
media_id = server_and_media_id.split("/")[1]
local_path = self.filepaths.local_media_filepath(media_id)
channel = make_request(
self.reactor,
FakeSite(download_resource),
"GET",
server_and_media_id,
shorthand=False,
access_token=self.admin_user_tok,
)
if expect_success:
self.assertEqual(
200,
channel.code,
msg=(
"Expected to receive a 200 on accessing media: %s"
% server_and_media_id
),
)
# Test that the file exists
self.assertTrue(os.path.exists(local_path))
else:
self.assertEqual(
404,
channel.code,
msg=(
"Expected to receive a 404 on accessing deleted media: %s"
% (server_and_media_id)
),
)
# Test that the file is deleted
self.assertFalse(os.path.exists(local_path))
class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase):
servlets = [
synapse.rest.admin.register_servlets,
synapse.rest.admin.register_servlets_for_media_repo,
login.register_servlets,
profile.register_servlets,
room.register_servlets,
]
def prepare(self, reactor: MemoryReactor, clock: Clock,
hs: HomeServer) -> None:
self.media_repo = hs.get_media_repository_resource()
self.server_name = hs.hostname
self.admin_user = self.register_user("admin", "pass", admin=True)
self.admin_user_tok = self.login("admin", "pass")
self.filepaths = MediaFilePaths(hs.config.media.media_store_path)
self.url = "/_synapse/admin/v1/media/%s/delete" % self.server_name
# Move clock up to somewhat realistic time
self.reactor.advance(1000000000)
def test_no_auth(self) -> None:
"""
Try to delete media without authentication.
"""
channel = self.make_request("POST", self.url, b"{}")
self.assertEqual(
HTTPStatus.UNAUTHORIZED,
channel.code,
msg=channel.json_body,
)
self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
def test_requester_is_no_admin(self) -> None:
"""
If the user is not a server admin, an error is returned.
"""
self.other_user = self.register_user("user", "pass")
self.other_user_token = self.login("user", "pass")
channel = self.make_request(
"POST",
self.url,
access_token=self.other_user_token,
)
self.assertEqual(
HTTPStatus.FORBIDDEN,
channel.code,
msg=channel.json_body,
)
self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
def test_media_is_not_local(self) -> None:
"""
Tests that a lookup for media that is not local returns a HTTPStatus.BAD_REQUEST
"""
url = "/_synapse/admin/v1/media/%s/delete" % "unknown_domain"
channel = self.make_request(
"POST",
url + f"?before_ts={VALID_TIMESTAMP}",
access_token=self.admin_user_tok,
)
self.assertEqual(HTTPStatus.BAD_REQUEST,
channel.code,
msg=channel.json_body)
self.assertEqual("Can only delete local media",
channel.json_body["error"])
def test_missing_parameter(self) -> None:
"""
If the parameter `before_ts` is missing, an error is returned.
"""
channel = self.make_request(
"POST",
self.url,
access_token=self.admin_user_tok,
)
self.assertEqual(
HTTPStatus.BAD_REQUEST,
channel.code,
msg=channel.json_body,
)
self.assertEqual(Codes.MISSING_PARAM, channel.json_body["errcode"])
self.assertEqual("Missing integer query parameter 'before_ts'",
channel.json_body["error"])
def test_invalid_parameter(self) -> None:
"""
If parameters are invalid, an error is returned.
"""
channel = self.make_request(
"POST",
self.url + "?before_ts=-1234",
access_token=self.admin_user_tok,
)
self.assertEqual(
HTTPStatus.BAD_REQUEST,
channel.code,
msg=channel.json_body,
)
self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
self.assertEqual(
"Query parameter before_ts must be a positive integer.",
channel.json_body["error"],
)
channel = self.make_request(
"POST",
self.url + f"?before_ts={INVALID_TIMESTAMP_IN_S}",
access_token=self.admin_user_tok,
)
self.assertEqual(
HTTPStatus.BAD_REQUEST,
channel.code,
msg=channel.json_body,
)
self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
self.assertEqual(
"Query parameter before_ts you provided is from the year 1970. " +
"Double check that you are providing a timestamp in milliseconds.",
channel.json_body["error"],
)
channel = self.make_request(
"POST",
self.url + f"?before_ts={VALID_TIMESTAMP}&size_gt=-1234",
access_token=self.admin_user_tok,
)
self.assertEqual(
HTTPStatus.BAD_REQUEST,
channel.code,
msg=channel.json_body,
)
self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
self.assertEqual(
"Query parameter size_gt must be a string representing a positive integer.",
channel.json_body["error"],
)
channel = self.make_request(
"POST",
self.url + f"?before_ts={VALID_TIMESTAMP}&keep_profiles=not_bool",
access_token=self.admin_user_tok,
)
self.assertEqual(
HTTPStatus.BAD_REQUEST,
channel.code,
msg=channel.json_body,
)
self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
self.assertEqual(
"Boolean query parameter 'keep_profiles' must be one of ['true', 'false']",
channel.json_body["error"],
)
def test_delete_media_never_accessed(self) -> None:
"""
Tests that media deleted if it is older than `before_ts` and never accessed
`last_access_ts` is `NULL` and `created_ts` < `before_ts`
"""
# upload and do not access
server_and_media_id = self._create_media()
self.pump(1.0)
# test that the file exists
media_id = server_and_media_id.split("/")[1]
local_path = self.filepaths.local_media_filepath(media_id)
self.assertTrue(os.path.exists(local_path))
# timestamp after upload/create
now_ms = self.clock.time_msec()
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms),
access_token=self.admin_user_tok,
)
self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
media_id,
channel.json_body["deleted_media"][0],
)
self._access_media(server_and_media_id, False)
def test_keep_media_by_date(self) -> None:
"""
Tests that media is not deleted if it is newer than `before_ts`
"""
# timestamp before upload
now_ms = self.clock.time_msec()
server_and_media_id = self._create_media()
self._access_media(server_and_media_id)
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms),
access_token=self.admin_user_tok,
)
self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
self.assertEqual(0, channel.json_body["total"])
self._access_media(server_and_media_id)
# timestamp after upload
now_ms = self.clock.time_msec()
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms),
access_token=self.admin_user_tok,
)
self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
server_and_media_id.split("/")[1],
channel.json_body["deleted_media"][0],
)
self._access_media(server_and_media_id, False)
def test_keep_media_by_size(self) -> None:
"""
Tests that media is not deleted if its size is smaller than or equal
to `size_gt`
"""
server_and_media_id = self._create_media()
self._access_media(server_and_media_id)
now_ms = self.clock.time_msec()
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&size_gt=67",
access_token=self.admin_user_tok,
)
self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
self.assertEqual(0, channel.json_body["total"])
self._access_media(server_and_media_id)
now_ms = self.clock.time_msec()
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&size_gt=66",
access_token=self.admin_user_tok,
)
self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
server_and_media_id.split("/")[1],
channel.json_body["deleted_media"][0],
)
self._access_media(server_and_media_id, False)
def test_keep_media_by_user_avatar(self) -> None:
"""
Tests that we do not delete media if is used as a user avatar
Tests parameter `keep_profiles`
"""
server_and_media_id = self._create_media()
self._access_media(server_and_media_id)
# set media as avatar
channel = self.make_request(
"PUT",
"/profile/%s/avatar_url" % (self.admin_user, ),
content={"avatar_url": "mxc://%s" % (server_and_media_id, )},
access_token=self.admin_user_tok,
)
self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
now_ms = self.clock.time_msec()
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=true",
access_token=self.admin_user_tok,
)
self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
self.assertEqual(0, channel.json_body["total"])
self._access_media(server_and_media_id)
now_ms = self.clock.time_msec()
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=false",
access_token=self.admin_user_tok,
)
self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
server_and_media_id.split("/")[1],
channel.json_body["deleted_media"][0],
)
self._access_media(server_and_media_id, False)
def test_keep_media_by_room_avatar(self) -> None:
"""
Tests that we do not delete media if it is used as a room avatar
Tests parameter `keep_profiles`
"""
server_and_media_id = self._create_media()
self._access_media(server_and_media_id)
# set media as room avatar
room_id = self.helper.create_room_as(self.admin_user,
tok=self.admin_user_tok)
channel = self.make_request(
"PUT",
"/rooms/%s/state/m.room.avatar" % (room_id, ),
content={"url": "mxc://%s" % (server_and_media_id, )},
access_token=self.admin_user_tok,
)
self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
now_ms = self.clock.time_msec()
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=true",
access_token=self.admin_user_tok,
)
self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
self.assertEqual(0, channel.json_body["total"])
self._access_media(server_and_media_id)
now_ms = self.clock.time_msec()
channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=false",
access_token=self.admin_user_tok,
)
self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
server_and_media_id.split("/")[1],
channel.json_body["deleted_media"][0],
)
self._access_media(server_and_media_id, False)
def _create_media(self) -> str:
"""
Create a media and return media_id and server_and_media_id
"""
upload_resource = self.media_repo.children[b"upload"]
# Upload some media into the room
response = self.helper.upload_media(
upload_resource,
SMALL_PNG,
tok=self.admin_user_tok,
expect_code=HTTPStatus.OK,
)
# Extract media ID from the response
server_and_media_id = response["content_uri"][6:] # Cut off 'mxc://'
server_name = server_and_media_id.split("/")[0]
# Check that new media is a local and not remote
self.assertEqual(server_name, self.server_name)
return server_and_media_id
def _access_media(self,
server_and_media_id: str,
expect_success: bool = True) -> None:
"""
Try to access a media and check the result
"""
download_resource = self.media_repo.children[b"download"]
media_id = server_and_media_id.split("/")[1]
local_path = self.filepaths.local_media_filepath(media_id)
channel = make_request(
self.reactor,
FakeSite(download_resource, self.reactor),
"GET",
server_and_media_id,
shorthand=False,
access_token=self.admin_user_tok,
)
if expect_success:
self.assertEqual(
HTTPStatus.OK,
channel.code,
msg=(
"Expected to receive a HTTPStatus.OK on accessing media: %s"
% server_and_media_id),
)
# Test that the file exists
self.assertTrue(os.path.exists(local_path))
else:
self.assertEqual(
HTTPStatus.NOT_FOUND,
channel.code,
msg=
("Expected to receive a HTTPStatus.NOT_FOUND on accessing deleted media: %s"
% (server_and_media_id)),
)
# Test that the file is deleted
self.assertFalse(os.path.exists(local_path))
