def prepare(self, reactor, clock, hs): self.media_repo = hs.get_media_repository_resource() self.server_name = hs.hostname self.admin_user = self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") self.filepaths = MediaFilePaths(hs.config.media_store_path)
def prepare(self, reactor, clock, hs): self.handler = hs.get_device_handler() self.media_repo = hs.get_media_repository_resource() self.server_name = hs.hostname self.admin_user = self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") self.filepaths = MediaFilePaths(hs.config.media_store_path) self.url = "/_synapse/admin/v1/media/%s/delete" % self.server_name
def main(src_repo, dest_repo): src_paths = MediaFilePaths(src_repo) dest_paths = MediaFilePaths(dest_repo) for line in sys.stdin: line = line.strip() parts = line.split("|") if len(parts) != 2: print("Unable to parse input line %s" % line, file=sys.stderr) exit(1) move_media(parts[0], parts[1], src_paths, dest_paths)
def prepare(self, reactor, clock, hs): self.media_repo = hs.get_media_repository_resource() self.server_name = hs.hostname self.admin_user = self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") self.filepaths = MediaFilePaths(hs.config.media.media_store_path) self.url = "/_synapse/admin/v1/media/%s/delete" % self.server_name # Move clock up to somewhat realistic time self.reactor.advance(1000000000)
def prepare(self, reactor, clock, hs): self.test_dir = tempfile.mkdtemp(prefix="synapse-tests-") self.addCleanup(shutil.rmtree, self.test_dir) self.primary_base_path = os.path.join(self.test_dir, "primary") self.secondary_base_path = os.path.join(self.test_dir, "secondary") hs.config.media_store_path = self.primary_base_path storage_providers = [FileStorageProviderBackend(hs, self.secondary_base_path)] self.filepaths = MediaFilePaths(self.primary_base_path) self.media_storage = MediaStorage( hs, self.primary_base_path, self.filepaths, storage_providers )
class MediaStorageTests(unittest.HomeserverTestCase): needs_threadpool = True def prepare(self, reactor, clock, hs): self.test_dir = tempfile.mkdtemp(prefix="synapse-tests-") self.addCleanup(shutil.rmtree, self.test_dir) self.primary_base_path = os.path.join(self.test_dir, "primary") self.secondary_base_path = os.path.join(self.test_dir, "secondary") hs.config.media_store_path = self.primary_base_path storage_providers = [ FileStorageProviderBackend(hs, self.secondary_base_path) ] self.filepaths = MediaFilePaths(self.primary_base_path) self.media_storage = MediaStorage(hs, self.primary_base_path, self.filepaths, storage_providers) def test_ensure_media_is_in_local_cache(self): media_id = "some_media_id" test_body = "Test\n" # First we create a file that is in a storage provider but not in the # local primary media store rel_path = self.filepaths.local_media_filepath_rel(media_id) secondary_path = os.path.join(self.secondary_base_path, rel_path) os.makedirs(os.path.dirname(secondary_path)) with open(secondary_path, "w") as f: f.write(test_body) # Now we run ensure_media_is_in_local_cache, which should copy the file # to the local cache. file_info = FileInfo(None, media_id) # This uses a real blocking threadpool so we have to wait for it to be # actually done :/ x = self.media_storage.ensure_media_is_in_local_cache(file_info) # Hotloop until the threadpool does its job... self.wait_on_thread(x) local_path = self.get_success(x) self.assertTrue(os.path.exists(local_path)) # Asserts the file is under the expected local cache directory self.assertEquals( os.path.commonprefix([self.primary_base_path, local_path]), self.primary_base_path, ) with open(local_path) as f: body = f.read() self.assertEqual(test_body, body)
def setUp(self): self.test_dir = tempfile.mkdtemp(prefix="synapse-tests-") self.primary_base_path = os.path.join(self.test_dir, "primary") self.secondary_base_path = os.path.join(self.test_dir, "secondary") hs = Mock() hs.get_reactor = Mock(return_value=reactor) hs.config.media_store_path = self.primary_base_path storage_providers = [FileStorageProviderBackend(hs, self.secondary_base_path)] self.filepaths = MediaFilePaths(self.primary_base_path) self.media_storage = MediaStorage( hs, self.primary_base_path, self.filepaths, storage_providers )
class MediaStorageTests(unittest.TestCase): def setUp(self): self.test_dir = tempfile.mkdtemp(prefix="synapse-tests-") self.primary_base_path = os.path.join(self.test_dir, "primary") self.secondary_base_path = os.path.join(self.test_dir, "secondary") hs = Mock() hs.get_reactor = Mock(return_value=reactor) hs.config.media_store_path = self.primary_base_path storage_providers = [ FileStorageProviderBackend(hs, self.secondary_base_path) ] self.filepaths = MediaFilePaths(self.primary_base_path) self.media_storage = MediaStorage( hs, self.primary_base_path, self.filepaths, storage_providers, ) def tearDown(self): shutil.rmtree(self.test_dir) @defer.inlineCallbacks def test_ensure_media_is_in_local_cache(self): media_id = "some_media_id" test_body = "Test\n" # First we create a file that is in a storage provider but not in the # local primary media store rel_path = self.filepaths.local_media_filepath_rel(media_id) secondary_path = os.path.join(self.secondary_base_path, rel_path) os.makedirs(os.path.dirname(secondary_path)) with open(secondary_path, "w") as f: f.write(test_body) # Now we run ensure_media_is_in_local_cache, which should copy the file # to the local cache. file_info = FileInfo(None, media_id) local_path = yield self.media_storage.ensure_media_is_in_local_cache( file_info) self.assertTrue(os.path.exists(local_path)) # Asserts the file is under the expected local cache directory self.assertEquals( os.path.commonprefix([self.primary_base_path, local_path]), self.primary_base_path, ) with open(local_path) as f: body = f.read() self.assertEqual(test_body, body)
def test_traversal_outside(self) -> None: """Test that the jail check fails for paths that escape the media directory.""" filepaths = MediaFilePaths("/media_store") path = "url_cache/2020-01-02/../../../GerZNDnDZVjsOtar" with self.assertRaises(ValueError): self._check_relative_path(filepaths, path) with self.assertRaises(ValueError): self._check_absolute_path(filepaths, path)
def test_traversal_inside(self) -> None: """Test the jail check for paths that stay within the media directory.""" # Despite the `../`s, these paths still lie within the media directory and it's # expected for the jail check to allow them through. # These paths ought to trip the other checks in place and should never be # returned. filepaths = MediaFilePaths("/media_store") path = "url_cache/2020-01-02/../../GerZNDnDZVjsOtar" self._check_relative_path(filepaths, path) self._check_absolute_path(filepaths, path)
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.media_repo = hs.get_media_repository_resource() self.server_name = hs.hostname self.admin_user = self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") self.filepaths = MediaFilePaths(hs.config.media.media_store_path) self.url = "/_synapse/admin/v1/purge_media_cache"
def test_traversal_reentry(self) -> None: """Test the jail check for paths that exit and re-enter the media directory.""" # These paths lie outside the media directory if it is a symlink, and inside # otherwise. Ideally the check should fail, but this proves difficult. # This test documents the behaviour for this edge case. # These paths ought to trip the other checks in place and should never be # returned. filepaths = MediaFilePaths("/media_store") path = "url_cache/2020-01-02/../../../media_store/GerZNDnDZVjsOtar" self._check_relative_path(filepaths, path) self._check_absolute_path(filepaths, path)
class MediaStorageTests(unittest.TestCase): def setUp(self): self.test_dir = tempfile.mkdtemp(prefix="synapse-tests-") self.primary_base_path = os.path.join(self.test_dir, "primary") self.secondary_base_path = os.path.join(self.test_dir, "secondary") hs = Mock() hs.config.media_store_path = self.primary_base_path storage_providers = [FileStorageProviderBackend( hs, self.secondary_base_path )] self.filepaths = MediaFilePaths(self.primary_base_path) self.media_storage = MediaStorage( self.primary_base_path, self.filepaths, storage_providers, ) def tearDown(self): shutil.rmtree(self.test_dir) @defer.inlineCallbacks def test_ensure_media_is_in_local_cache(self): media_id = "some_media_id" test_body = "Test\n" # First we create a file that is in a storage provider but not in the # local primary media store rel_path = self.filepaths.local_media_filepath_rel(media_id) secondary_path = os.path.join(self.secondary_base_path, rel_path) os.makedirs(os.path.dirname(secondary_path)) with open(secondary_path, "w") as f: f.write(test_body) # Now we run ensure_media_is_in_local_cache, which should copy the file # to the local cache. file_info = FileInfo(None, media_id) local_path = yield self.media_storage.ensure_media_is_in_local_cache(file_info) self.assertTrue(os.path.exists(local_path)) # Asserts the file is under the expected local cache directory self.assertEquals( os.path.commonprefix([self.primary_base_path, local_path]), self.primary_base_path, ) with open(local_path) as f: body = f.read() self.assertEqual(test_body, body)
def test_symlink(self) -> None: """Test that a symlink does not cause the jail check to fail.""" media_store_path = self.mktemp() # symlink the media store directory os.symlink("/mnt/synapse/media_store", media_store_path) # Test that relative and absolute paths don't trip the check # NB: `media_store_path` is a relative path filepaths = MediaFilePaths(media_store_path) self._check_relative_path(filepaths, "url_cache/2020-01-02/GerZNDnDZVjsOtar") self._check_absolute_path(filepaths, "url_cache/2020-01-02/GerZNDnDZVjsOtar") filepaths = MediaFilePaths(os.path.abspath(media_store_path)) self._check_relative_path(filepaths, "url_cache/2020-01-02/GerZNDnDZVjsOtar") self._check_absolute_path(filepaths, "url_cache/2020-01-02/GerZNDnDZVjsOtar")
def move_media( origin_server: str, file_id: str, src_paths: MediaFilePaths, dest_paths: MediaFilePaths, ) -> None: """Move the given file, and any thumbnails, to the dest repo Args: origin_server: file_id: src_paths: dest_paths: """ logger.info("%s/%s", origin_server, file_id) # check that the original exists original_file = src_paths.remote_media_filepath(origin_server, file_id) if not os.path.exists(original_file): logger.warning( "Original for %s/%s (%s) does not exist", origin_server, file_id, original_file, ) else: mkdir_and_move( original_file, dest_paths.remote_media_filepath(origin_server, file_id)) # now look for thumbnails original_thumb_dir = src_paths.remote_media_thumbnail_dir( origin_server, file_id) if not os.path.exists(original_thumb_dir): return mkdir_and_move( original_thumb_dir, dest_paths.remote_media_thumbnail_dir(origin_server, file_id), )
class MediaFilePathsTestCase(unittest.TestCase): def setUp(self): super().setUp() self.filepaths = MediaFilePaths("/media_store") def test_local_media_filepath(self): """Test local media paths""" self.assertEqual( self.filepaths.local_media_filepath_rel("GerZNDnDZVjsOtardLuwfIBg"), "local_content/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) self.assertEqual( self.filepaths.local_media_filepath("GerZNDnDZVjsOtardLuwfIBg"), "/media_store/local_content/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) def test_local_media_thumbnail(self): """Test local media thumbnail paths""" self.assertEqual( self.filepaths.local_media_thumbnail_rel( "GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale" ), "local_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale", ) self.assertEqual( self.filepaths.local_media_thumbnail( "GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale" ), "/media_store/local_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale", ) def test_local_media_thumbnail_dir(self): """Test local media thumbnail directory paths""" self.assertEqual( self.filepaths.local_media_thumbnail_dir("GerZNDnDZVjsOtardLuwfIBg"), "/media_store/local_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) def test_remote_media_filepath(self): """Test remote media paths""" self.assertEqual( self.filepaths.remote_media_filepath_rel( "example.com", "GerZNDnDZVjsOtardLuwfIBg" ), "remote_content/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) self.assertEqual( self.filepaths.remote_media_filepath( "example.com", "GerZNDnDZVjsOtardLuwfIBg" ), "/media_store/remote_content/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) def test_remote_media_thumbnail(self): """Test remote media thumbnail paths""" self.assertEqual( self.filepaths.remote_media_thumbnail_rel( "example.com", "GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale", ), "remote_thumbnail/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale", ) self.assertEqual( self.filepaths.remote_media_thumbnail( "example.com", "GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale", ), "/media_store/remote_thumbnail/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale", ) def test_remote_media_thumbnail_legacy(self): """Test old-style remote media thumbnail paths""" self.assertEqual( self.filepaths.remote_media_thumbnail_rel_legacy( "example.com", "GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg" ), "remote_thumbnail/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg", ) def test_remote_media_thumbnail_dir(self): """Test remote media thumbnail directory paths""" self.assertEqual( self.filepaths.remote_media_thumbnail_dir( "example.com", "GerZNDnDZVjsOtardLuwfIBg" ), "/media_store/remote_thumbnail/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) def test_url_cache_filepath(self): """Test URL cache paths""" self.assertEqual( self.filepaths.url_cache_filepath_rel("2020-01-02_GerZNDnDZVjsOtar"), "url_cache/2020-01-02/GerZNDnDZVjsOtar", ) self.assertEqual( self.filepaths.url_cache_filepath("2020-01-02_GerZNDnDZVjsOtar"), "/media_store/url_cache/2020-01-02/GerZNDnDZVjsOtar", ) def test_url_cache_filepath_legacy(self): """Test old-style URL cache paths""" self.assertEqual( self.filepaths.url_cache_filepath_rel("GerZNDnDZVjsOtardLuwfIBg"), "url_cache/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) self.assertEqual( self.filepaths.url_cache_filepath("GerZNDnDZVjsOtardLuwfIBg"), "/media_store/url_cache/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) def test_url_cache_filepath_dirs_to_delete(self): """Test URL cache cleanup paths""" self.assertEqual( self.filepaths.url_cache_filepath_dirs_to_delete( "2020-01-02_GerZNDnDZVjsOtar" ), ["/media_store/url_cache/2020-01-02"], ) def test_url_cache_filepath_dirs_to_delete_legacy(self): """Test old-style URL cache cleanup paths""" self.assertEqual( self.filepaths.url_cache_filepath_dirs_to_delete( "GerZNDnDZVjsOtardLuwfIBg" ), [ "/media_store/url_cache/Ge/rZ", "/media_store/url_cache/Ge", ], ) def test_url_cache_thumbnail(self): """Test URL cache thumbnail paths""" self.assertEqual( self.filepaths.url_cache_thumbnail_rel( "2020-01-02_GerZNDnDZVjsOtar", 800, 600, "image/jpeg", "scale" ), "url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar/800-600-image-jpeg-scale", ) self.assertEqual( self.filepaths.url_cache_thumbnail( "2020-01-02_GerZNDnDZVjsOtar", 800, 600, "image/jpeg", "scale" ), "/media_store/url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar/800-600-image-jpeg-scale", ) def test_url_cache_thumbnail_legacy(self): """Test old-style URL cache thumbnail paths""" self.assertEqual( self.filepaths.url_cache_thumbnail_rel( "GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale" ), "url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale", ) self.assertEqual( self.filepaths.url_cache_thumbnail( "GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale" ), "/media_store/url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale", ) def test_url_cache_thumbnail_directory(self): """Test URL cache thumbnail directory paths""" self.assertEqual( self.filepaths.url_cache_thumbnail_directory_rel( "2020-01-02_GerZNDnDZVjsOtar" ), "url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar", ) self.assertEqual( self.filepaths.url_cache_thumbnail_directory("2020-01-02_GerZNDnDZVjsOtar"), "/media_store/url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar", ) def test_url_cache_thumbnail_directory_legacy(self): """Test old-style URL cache thumbnail directory paths""" self.assertEqual( self.filepaths.url_cache_thumbnail_directory_rel( "GerZNDnDZVjsOtardLuwfIBg" ), "url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) self.assertEqual( self.filepaths.url_cache_thumbnail_directory("GerZNDnDZVjsOtardLuwfIBg"), "/media_store/url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) def test_url_cache_thumbnail_dirs_to_delete(self): """Test URL cache thumbnail cleanup paths""" self.assertEqual( self.filepaths.url_cache_thumbnail_dirs_to_delete( "2020-01-02_GerZNDnDZVjsOtar" ), [ "/media_store/url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar", "/media_store/url_cache_thumbnails/2020-01-02", ], ) def test_url_cache_thumbnail_dirs_to_delete_legacy(self): """Test old-style URL cache thumbnail cleanup paths""" self.assertEqual( self.filepaths.url_cache_thumbnail_dirs_to_delete( "GerZNDnDZVjsOtardLuwfIBg" ), [ "/media_store/url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg", "/media_store/url_cache_thumbnails/Ge/rZ", "/media_store/url_cache_thumbnails/Ge", ], )
class DeleteMediaByIDTestCase(unittest.HomeserverTestCase): servlets = [ synapse.rest.admin.register_servlets, synapse.rest.admin.register_servlets_for_media_repo, login.register_servlets, ] def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.media_repo = hs.get_media_repository_resource() self.server_name = hs.hostname self.admin_user = self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") self.filepaths = MediaFilePaths(hs.config.media.media_store_path) def test_no_auth(self) -> None: """ Try to delete media without authentication. """ url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345") channel = self.make_request("DELETE", url, b"{}") self.assertEqual( HTTPStatus.UNAUTHORIZED, channel.code, msg=channel.json_body, ) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) def test_requester_is_no_admin(self) -> None: """ If the user is not a server admin, an error is returned. """ self.other_user = self.register_user("user", "pass") self.other_user_token = self.login("user", "pass") url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345") channel = self.make_request( "DELETE", url, access_token=self.other_user_token, ) self.assertEqual( HTTPStatus.FORBIDDEN, channel.code, msg=channel.json_body, ) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) def test_media_does_not_exist(self) -> None: """ Tests that a lookup for a media that does not exist returns a HTTPStatus.NOT_FOUND """ url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345") channel = self.make_request( "DELETE", url, access_token=self.admin_user_tok, ) self.assertEqual(HTTPStatus.NOT_FOUND, channel.code, msg=channel.json_body) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) def test_media_is_not_local(self) -> None: """ Tests that a lookup for a media that is not a local returns a HTTPStatus.BAD_REQUEST """ url = "/_synapse/admin/v1/media/%s/%s" % ("unknown_domain", "12345") channel = self.make_request( "DELETE", url, access_token=self.admin_user_tok, ) self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, msg=channel.json_body) self.assertEqual("Can only delete local media", channel.json_body["error"]) def test_delete_media(self) -> None: """ Tests that delete a media is successfully """ download_resource = self.media_repo.children[b"download"] upload_resource = self.media_repo.children[b"upload"] # Upload some media into the room response = self.helper.upload_media( upload_resource, SMALL_PNG, tok=self.admin_user_tok, expect_code=HTTPStatus.OK, ) # Extract media ID from the response server_and_media_id = response["content_uri"][6:] # Cut off 'mxc://' server_name, media_id = server_and_media_id.split("/") self.assertEqual(server_name, self.server_name) # Attempt to access media channel = make_request( self.reactor, FakeSite(download_resource, self.reactor), "GET", server_and_media_id, shorthand=False, access_token=self.admin_user_tok, ) # Should be successful self.assertEqual( HTTPStatus.OK, channel.code, msg=("Expected to receive a HTTPStatus.OK on accessing media: %s" % server_and_media_id), ) # Test if the file exists local_path = self.filepaths.local_media_filepath(media_id) self.assertTrue(os.path.exists(local_path)) url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, media_id) # Delete media channel = self.make_request( "DELETE", url, access_token=self.admin_user_tok, ) self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( media_id, channel.json_body["deleted_media"][0], ) # Attempt to access media channel = make_request( self.reactor, FakeSite(download_resource, self.reactor), "GET", server_and_media_id, shorthand=False, access_token=self.admin_user_tok, ) self.assertEqual( HTTPStatus.NOT_FOUND, channel.code, msg= ("Expected to receive a HTTPStatus.NOT_FOUND on accessing deleted media: %s" % server_and_media_id), ) # Test if the file is deleted self.assertFalse(os.path.exists(local_path))
class MediaFilePathsTestCase(unittest.TestCase): def setUp(self): super().setUp() self.filepaths = MediaFilePaths("/media_store") def test_local_media_filepath(self): """Test local media paths""" self.assertEqual( self.filepaths.local_media_filepath_rel( "GerZNDnDZVjsOtardLuwfIBg"), "local_content/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) self.assertEqual( self.filepaths.local_media_filepath("GerZNDnDZVjsOtardLuwfIBg"), "/media_store/local_content/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) def test_local_media_thumbnail(self): """Test local media thumbnail paths""" self.assertEqual( self.filepaths.local_media_thumbnail_rel( "GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale"), "local_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale", ) self.assertEqual( self.filepaths.local_media_thumbnail("GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale"), "/media_store/local_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale", ) def test_local_media_thumbnail_dir(self): """Test local media thumbnail directory paths""" self.assertEqual( self.filepaths.local_media_thumbnail_dir( "GerZNDnDZVjsOtardLuwfIBg"), "/media_store/local_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) def test_remote_media_filepath(self): """Test remote media paths""" self.assertEqual( self.filepaths.remote_media_filepath_rel( "example.com", "GerZNDnDZVjsOtardLuwfIBg"), "remote_content/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) self.assertEqual( self.filepaths.remote_media_filepath("example.com", "GerZNDnDZVjsOtardLuwfIBg"), "/media_store/remote_content/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) def test_remote_media_thumbnail(self): """Test remote media thumbnail paths""" self.assertEqual( self.filepaths.remote_media_thumbnail_rel( "example.com", "GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale", ), "remote_thumbnail/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale", ) self.assertEqual( self.filepaths.remote_media_thumbnail( "example.com", "GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale", ), "/media_store/remote_thumbnail/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale", ) def test_remote_media_thumbnail_legacy(self): """Test old-style remote media thumbnail paths""" self.assertEqual( self.filepaths.remote_media_thumbnail_rel_legacy( "example.com", "GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg"), "remote_thumbnail/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg", ) def test_remote_media_thumbnail_dir(self): """Test remote media thumbnail directory paths""" self.assertEqual( self.filepaths.remote_media_thumbnail_dir( "example.com", "GerZNDnDZVjsOtardLuwfIBg"), "/media_store/remote_thumbnail/example.com/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) def test_url_cache_filepath(self): """Test URL cache paths""" self.assertEqual( self.filepaths.url_cache_filepath_rel( "2020-01-02_GerZNDnDZVjsOtar"), "url_cache/2020-01-02/GerZNDnDZVjsOtar", ) self.assertEqual( self.filepaths.url_cache_filepath("2020-01-02_GerZNDnDZVjsOtar"), "/media_store/url_cache/2020-01-02/GerZNDnDZVjsOtar", ) def test_url_cache_filepath_legacy(self): """Test old-style URL cache paths""" self.assertEqual( self.filepaths.url_cache_filepath_rel("GerZNDnDZVjsOtardLuwfIBg"), "url_cache/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) self.assertEqual( self.filepaths.url_cache_filepath("GerZNDnDZVjsOtardLuwfIBg"), "/media_store/url_cache/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) def test_url_cache_filepath_dirs_to_delete(self): """Test URL cache cleanup paths""" self.assertEqual( self.filepaths.url_cache_filepath_dirs_to_delete( "2020-01-02_GerZNDnDZVjsOtar"), ["/media_store/url_cache/2020-01-02"], ) def test_url_cache_filepath_dirs_to_delete_legacy(self): """Test old-style URL cache cleanup paths""" self.assertEqual( self.filepaths.url_cache_filepath_dirs_to_delete( "GerZNDnDZVjsOtardLuwfIBg"), [ "/media_store/url_cache/Ge/rZ", "/media_store/url_cache/Ge", ], ) def test_url_cache_thumbnail(self): """Test URL cache thumbnail paths""" self.assertEqual( self.filepaths.url_cache_thumbnail_rel( "2020-01-02_GerZNDnDZVjsOtar", 800, 600, "image/jpeg", "scale"), "url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar/800-600-image-jpeg-scale", ) self.assertEqual( self.filepaths.url_cache_thumbnail("2020-01-02_GerZNDnDZVjsOtar", 800, 600, "image/jpeg", "scale"), "/media_store/url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar/800-600-image-jpeg-scale", ) def test_url_cache_thumbnail_legacy(self): """Test old-style URL cache thumbnail paths""" self.assertEqual( self.filepaths.url_cache_thumbnail_rel("GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale"), "url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale", ) self.assertEqual( self.filepaths.url_cache_thumbnail("GerZNDnDZVjsOtardLuwfIBg", 800, 600, "image/jpeg", "scale"), "/media_store/url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg/800-600-image-jpeg-scale", ) def test_url_cache_thumbnail_directory(self): """Test URL cache thumbnail directory paths""" self.assertEqual( self.filepaths.url_cache_thumbnail_directory_rel( "2020-01-02_GerZNDnDZVjsOtar"), "url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar", ) self.assertEqual( self.filepaths.url_cache_thumbnail_directory( "2020-01-02_GerZNDnDZVjsOtar"), "/media_store/url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar", ) def test_url_cache_thumbnail_directory_legacy(self): """Test old-style URL cache thumbnail directory paths""" self.assertEqual( self.filepaths.url_cache_thumbnail_directory_rel( "GerZNDnDZVjsOtardLuwfIBg"), "url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) self.assertEqual( self.filepaths.url_cache_thumbnail_directory( "GerZNDnDZVjsOtardLuwfIBg"), "/media_store/url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg", ) def test_url_cache_thumbnail_dirs_to_delete(self): """Test URL cache thumbnail cleanup paths""" self.assertEqual( self.filepaths.url_cache_thumbnail_dirs_to_delete( "2020-01-02_GerZNDnDZVjsOtar"), [ "/media_store/url_cache_thumbnails/2020-01-02/GerZNDnDZVjsOtar", "/media_store/url_cache_thumbnails/2020-01-02", ], ) def test_url_cache_thumbnail_dirs_to_delete_legacy(self): """Test old-style URL cache thumbnail cleanup paths""" self.assertEqual( self.filepaths.url_cache_thumbnail_dirs_to_delete( "GerZNDnDZVjsOtardLuwfIBg"), [ "/media_store/url_cache_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg", "/media_store/url_cache_thumbnails/Ge/rZ", "/media_store/url_cache_thumbnails/Ge", ], ) def test_server_name_validation(self): """Test validation of server names""" self._test_path_validation( [ "remote_media_filepath_rel", "remote_media_filepath", "remote_media_thumbnail_rel", "remote_media_thumbnail", "remote_media_thumbnail_rel_legacy", "remote_media_thumbnail_dir", ], parameter="server_name", valid_values=[ "matrix.org", "matrix.org:8448", "matrix-federation.matrix.org", "matrix-federation.matrix.org:8448", "10.1.12.123", "10.1.12.123:8448", "[fd00:abcd::ffff]", "[fd00:abcd::ffff]:8448", ], invalid_values=[ "/matrix.org", "matrix.org/..", "matrix.org\x00", "", ".", "..", "/", ], ) def test_file_id_validation(self): """Test validation of local, remote and legacy URL cache file / media IDs""" # File / media IDs get split into three parts to form paths, consisting of the # first two characters, next two characters and rest of the ID. valid_file_ids = [ "GerZNDnDZVjsOtardLuwfIBg", # Unexpected, but produces an acceptable path: "GerZN", # "N" becomes the last directory ] invalid_file_ids = [ "/erZNDnDZVjsOtardLuwfIBg", "Ge/ZNDnDZVjsOtardLuwfIBg", "GerZ/DnDZVjsOtardLuwfIBg", "GerZ/..", "G\x00rZNDnDZVjsOtardLuwfIBg", "Ger\x00NDnDZVjsOtardLuwfIBg", "GerZNDnDZVjsOtardLuwfIBg\x00", "", "Ge", "GerZ", "GerZ.", "..rZNDnDZVjsOtardLuwfIBg", "Ge..NDnDZVjsOtardLuwfIBg", "GerZ..", "GerZ/", ] self._test_path_validation( [ "local_media_filepath_rel", "local_media_filepath", "local_media_thumbnail_rel", "local_media_thumbnail", "local_media_thumbnail_dir", # Legacy URL cache media IDs "url_cache_filepath_rel", "url_cache_filepath", # `url_cache_filepath_dirs_to_delete` is tested below. "url_cache_thumbnail_rel", "url_cache_thumbnail", "url_cache_thumbnail_directory_rel", "url_cache_thumbnail_directory", "url_cache_thumbnail_dirs_to_delete", ], parameter="media_id", valid_values=valid_file_ids, invalid_values=invalid_file_ids, ) # `url_cache_filepath_dirs_to_delete` ignores what would be the last path # component, so only the first 4 characters matter. self._test_path_validation( [ "url_cache_filepath_dirs_to_delete", ], parameter="media_id", valid_values=valid_file_ids, invalid_values=[ "/erZNDnDZVjsOtardLuwfIBg", "Ge/ZNDnDZVjsOtardLuwfIBg", "G\x00rZNDnDZVjsOtardLuwfIBg", "Ger\x00NDnDZVjsOtardLuwfIBg", "", "Ge", "..rZNDnDZVjsOtardLuwfIBg", "Ge..NDnDZVjsOtardLuwfIBg", ], ) self._test_path_validation( [ "remote_media_filepath_rel", "remote_media_filepath", "remote_media_thumbnail_rel", "remote_media_thumbnail", "remote_media_thumbnail_rel_legacy", "remote_media_thumbnail_dir", ], parameter="file_id", valid_values=valid_file_ids, invalid_values=invalid_file_ids, ) def test_url_cache_media_id_validation(self): """Test validation of URL cache media IDs""" self._test_path_validation( [ "url_cache_filepath_rel", "url_cache_filepath", # `url_cache_filepath_dirs_to_delete` only cares about the date prefix "url_cache_thumbnail_rel", "url_cache_thumbnail", "url_cache_thumbnail_directory_rel", "url_cache_thumbnail_directory", "url_cache_thumbnail_dirs_to_delete", ], parameter="media_id", valid_values=[ "2020-01-02_GerZNDnDZVjsOtar", "2020-01-02_G", # Unexpected, but produces an acceptable path ], invalid_values=[ "2020-01-02", "2020-01-02-", "2020-01-02-.", "2020-01-02-..", "2020-01-02-/", "2020-01-02-/GerZNDnDZVjsOtar", "2020-01-02-GerZNDnDZVjsOtar/..", "2020-01-02-GerZNDnDZVjsOtar\x00", ], ) def test_content_type_validation(self): """Test validation of thumbnail content types""" self._test_path_validation( [ "local_media_thumbnail_rel", "local_media_thumbnail", "remote_media_thumbnail_rel", "remote_media_thumbnail", "remote_media_thumbnail_rel_legacy", "url_cache_thumbnail_rel", "url_cache_thumbnail", ], parameter="content_type", valid_values=[ "image/jpeg", ], invalid_values=[ "", # ValueError: not enough values to unpack "image/jpeg/abc", # ValueError: too many values to unpack "image/jpeg\x00", ], ) def test_thumbnail_method_validation(self): """Test validation of thumbnail methods""" self._test_path_validation( [ "local_media_thumbnail_rel", "local_media_thumbnail", "remote_media_thumbnail_rel", "remote_media_thumbnail", "url_cache_thumbnail_rel", "url_cache_thumbnail", ], parameter="method", valid_values=[ "crop", "scale", ], invalid_values=[ "/scale", "scale/..", "scale\x00", "/", ], ) def _test_path_validation( self, methods: Iterable[str], parameter: str, valid_values: Iterable[str], invalid_values: Iterable[str], ): """Test that the specified methods validate the named parameter as expected Args: methods: The names of `MediaFilePaths` methods to test parameter: The name of the parameter to test valid_values: A list of parameter values that are expected to be accepted invalid_values: A list of parameter values that are expected to be rejected Raises: AssertionError: If a value was accepted when it should have failed validation. ValueError: If a value failed validation when it should have been accepted. """ for method in methods: get_path = getattr(self.filepaths, method) parameters = inspect.signature(get_path).parameters kwargs = { "server_name": "matrix.org", "media_id": "GerZNDnDZVjsOtardLuwfIBg", "file_id": "GerZNDnDZVjsOtardLuwfIBg", "width": 800, "height": 600, "content_type": "image/jpeg", "method": "scale", } if get_path.__name__.startswith("url_"): kwargs["media_id"] = "2020-01-02_GerZNDnDZVjsOtar" kwargs = {k: v for k, v in kwargs.items() if k in parameters} kwargs.pop(parameter) for value in valid_values: kwargs[parameter] = value get_path(**kwargs) # No exception should be raised for value in invalid_values: with self.assertRaises(ValueError): kwargs[parameter] = value path_or_list = get_path(**kwargs) self.fail(f"{value!r} unexpectedly passed validation: " f"{method} returned {path_or_list!r}")
def setUp(self): super().setUp() self.filepaths = MediaFilePaths("/media_store")
class DeleteMediaByIDTestCase(unittest.HomeserverTestCase): servlets = [ synapse.rest.admin.register_servlets, synapse.rest.admin.register_servlets_for_media_repo, login.register_servlets, ] def prepare(self, reactor, clock, hs): self.media_repo = hs.get_media_repository_resource() self.server_name = hs.hostname self.admin_user = self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") self.filepaths = MediaFilePaths(hs.config.media_store_path) def test_no_auth(self): """ Try to delete media without authentication. """ url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345") channel = self.make_request("DELETE", url, b"{}") self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) def test_requester_is_no_admin(self): """ If the user is not a server admin, an error is returned. """ self.other_user = self.register_user("user", "pass") self.other_user_token = self.login("user", "pass") url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345") channel = self.make_request( "DELETE", url, access_token=self.other_user_token, ) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) def test_media_does_not_exist(self): """ Tests that a lookup for a media that does not exist returns a 404 """ url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345") channel = self.make_request( "DELETE", url, access_token=self.admin_user_tok, ) self.assertEqual(404, channel.code, msg=channel.json_body) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) def test_media_is_not_local(self): """ Tests that a lookup for a media that is not a local returns a 400 """ url = "/_synapse/admin/v1/media/%s/%s" % ("unknown_domain", "12345") channel = self.make_request( "DELETE", url, access_token=self.admin_user_tok, ) self.assertEqual(400, channel.code, msg=channel.json_body) self.assertEqual("Can only delete local media", channel.json_body["error"]) def test_delete_media(self): """ Tests that delete a media is successfully """ download_resource = self.media_repo.children[b"download"] upload_resource = self.media_repo.children[b"upload"] image_data = unhexlify( b"89504e470d0a1a0a0000000d4948445200000001000000010806" b"0000001f15c4890000000a49444154789c63000100000500010d" b"0a2db40000000049454e44ae426082" ) # Upload some media into the room response = self.helper.upload_media( upload_resource, image_data, tok=self.admin_user_tok, expect_code=200 ) # Extract media ID from the response server_and_media_id = response["content_uri"][6:] # Cut off 'mxc://' server_name, media_id = server_and_media_id.split("/") self.assertEqual(server_name, self.server_name) # Attempt to access media channel = make_request( self.reactor, FakeSite(download_resource), "GET", server_and_media_id, shorthand=False, access_token=self.admin_user_tok, ) # Should be successful self.assertEqual( 200, channel.code, msg=( "Expected to receive a 200 on accessing media: %s" % server_and_media_id ), ) # Test if the file exists local_path = self.filepaths.local_media_filepath(media_id) self.assertTrue(os.path.exists(local_path)) url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, media_id) # Delete media channel = self.make_request( "DELETE", url, access_token=self.admin_user_tok, ) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( media_id, channel.json_body["deleted_media"][0], ) # Attempt to access media channel = make_request( self.reactor, FakeSite(download_resource), "GET", server_and_media_id, shorthand=False, access_token=self.admin_user_tok, ) self.assertEqual( 404, channel.code, msg=( "Expected to receive a 404 on accessing deleted media: %s" % server_and_media_id ), ) # Test if the file is deleted self.assertFalse(os.path.exists(local_path))
class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): servlets = [ synapse.rest.admin.register_servlets, synapse.rest.admin.register_servlets_for_media_repo, login.register_servlets, profile.register_servlets, room.register_servlets, ] def prepare(self, reactor, clock, hs): self.media_repo = hs.get_media_repository_resource() self.server_name = hs.hostname self.admin_user = self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") self.filepaths = MediaFilePaths(hs.config.media_store_path) self.url = "/_synapse/admin/v1/media/%s/delete" % self.server_name def test_no_auth(self): """ Try to delete media without authentication. """ channel = self.make_request("POST", self.url, b"{}") self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) def test_requester_is_no_admin(self): """ If the user is not a server admin, an error is returned. """ self.other_user = self.register_user("user", "pass") self.other_user_token = self.login("user", "pass") channel = self.make_request( "POST", self.url, access_token=self.other_user_token, ) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) def test_media_is_not_local(self): """ Tests that a lookup for media that is not local returns a 400 """ url = "/_synapse/admin/v1/media/%s/delete" % "unknown_domain" channel = self.make_request( "POST", url + "?before_ts=1234", access_token=self.admin_user_tok, ) self.assertEqual(400, channel.code, msg=channel.json_body) self.assertEqual("Can only delete local media", channel.json_body["error"]) def test_missing_parameter(self): """ If the parameter `before_ts` is missing, an error is returned. """ channel = self.make_request( "POST", self.url, access_token=self.admin_user_tok, ) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_PARAM, channel.json_body["errcode"]) self.assertEqual( "Missing integer query parameter b'before_ts'", channel.json_body["error"] ) def test_invalid_parameter(self): """ If parameters are invalid, an error is returned. """ channel = self.make_request( "POST", self.url + "?before_ts=-1234", access_token=self.admin_user_tok, ) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) self.assertEqual( "Query parameter before_ts must be a string representing a positive integer.", channel.json_body["error"], ) channel = self.make_request( "POST", self.url + "?before_ts=1234&size_gt=-1234", access_token=self.admin_user_tok, ) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) self.assertEqual( "Query parameter size_gt must be a string representing a positive integer.", channel.json_body["error"], ) channel = self.make_request( "POST", self.url + "?before_ts=1234&keep_profiles=not_bool", access_token=self.admin_user_tok, ) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.UNKNOWN, channel.json_body["errcode"]) self.assertEqual( "Boolean query parameter b'keep_profiles' must be one of ['true', 'false']", channel.json_body["error"], ) def test_delete_media_never_accessed(self): """ Tests that media deleted if it is older than `before_ts` and never accessed `last_access_ts` is `NULL` and `created_ts` < `before_ts` """ # upload and do not access server_and_media_id = self._create_media() self.pump(1.0) # test that the file exists media_id = server_and_media_id.split("/")[1] local_path = self.filepaths.local_media_filepath(media_id) self.assertTrue(os.path.exists(local_path)) # timestamp after upload/create now_ms = self.clock.time_msec() channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms), access_token=self.admin_user_tok, ) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( media_id, channel.json_body["deleted_media"][0], ) self._access_media(server_and_media_id, False) def test_keep_media_by_date(self): """ Tests that media is not deleted if it is newer than `before_ts` """ # timestamp before upload now_ms = self.clock.time_msec() server_and_media_id = self._create_media() self._access_media(server_and_media_id) channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms), access_token=self.admin_user_tok, ) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) self._access_media(server_and_media_id) # timestamp after upload now_ms = self.clock.time_msec() channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms), access_token=self.admin_user_tok, ) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( server_and_media_id.split("/")[1], channel.json_body["deleted_media"][0], ) self._access_media(server_and_media_id, False) def test_keep_media_by_size(self): """ Tests that media is not deleted if its size is smaller than or equal to `size_gt` """ server_and_media_id = self._create_media() self._access_media(server_and_media_id) now_ms = self.clock.time_msec() channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms) + "&size_gt=67", access_token=self.admin_user_tok, ) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) self._access_media(server_and_media_id) now_ms = self.clock.time_msec() channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms) + "&size_gt=66", access_token=self.admin_user_tok, ) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( server_and_media_id.split("/")[1], channel.json_body["deleted_media"][0], ) self._access_media(server_and_media_id, False) def test_keep_media_by_user_avatar(self): """ Tests that we do not delete media if is used as a user avatar Tests parameter `keep_profiles` """ server_and_media_id = self._create_media() self._access_media(server_and_media_id) # set media as avatar channel = self.make_request( "PUT", "/profile/%s/avatar_url" % (self.admin_user,), content=json.dumps({"avatar_url": "mxc://%s" % (server_and_media_id,)}), access_token=self.admin_user_tok, ) self.assertEqual(200, channel.code, msg=channel.json_body) now_ms = self.clock.time_msec() channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=true", access_token=self.admin_user_tok, ) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) self._access_media(server_and_media_id) now_ms = self.clock.time_msec() channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=false", access_token=self.admin_user_tok, ) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( server_and_media_id.split("/")[1], channel.json_body["deleted_media"][0], ) self._access_media(server_and_media_id, False) def test_keep_media_by_room_avatar(self): """ Tests that we do not delete media if it is used as a room avatar Tests parameter `keep_profiles` """ server_and_media_id = self._create_media() self._access_media(server_and_media_id) # set media as room avatar room_id = self.helper.create_room_as(self.admin_user, tok=self.admin_user_tok) channel = self.make_request( "PUT", "/rooms/%s/state/m.room.avatar" % (room_id,), content=json.dumps({"url": "mxc://%s" % (server_and_media_id,)}), access_token=self.admin_user_tok, ) self.assertEqual(200, channel.code, msg=channel.json_body) now_ms = self.clock.time_msec() channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=true", access_token=self.admin_user_tok, ) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) self._access_media(server_and_media_id) now_ms = self.clock.time_msec() channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=false", access_token=self.admin_user_tok, ) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( server_and_media_id.split("/")[1], channel.json_body["deleted_media"][0], ) self._access_media(server_and_media_id, False) def _create_media(self): """ Create a media and return media_id and server_and_media_id """ upload_resource = self.media_repo.children[b"upload"] # file size is 67 Byte image_data = unhexlify( b"89504e470d0a1a0a0000000d4948445200000001000000010806" b"0000001f15c4890000000a49444154789c63000100000500010d" b"0a2db40000000049454e44ae426082" ) # Upload some media into the room response = self.helper.upload_media( upload_resource, image_data, tok=self.admin_user_tok, expect_code=200 ) # Extract media ID from the response server_and_media_id = response["content_uri"][6:] # Cut off 'mxc://' server_name = server_and_media_id.split("/")[0] # Check that new media is a local and not remote self.assertEqual(server_name, self.server_name) return server_and_media_id def _access_media(self, server_and_media_id, expect_success=True): """ Try to access a media and check the result """ download_resource = self.media_repo.children[b"download"] media_id = server_and_media_id.split("/")[1] local_path = self.filepaths.local_media_filepath(media_id) channel = make_request( self.reactor, FakeSite(download_resource), "GET", server_and_media_id, shorthand=False, access_token=self.admin_user_tok, ) if expect_success: self.assertEqual( 200, channel.code, msg=( "Expected to receive a 200 on accessing media: %s" % server_and_media_id ), ) # Test that the file exists self.assertTrue(os.path.exists(local_path)) else: self.assertEqual( 404, channel.code, msg=( "Expected to receive a 404 on accessing deleted media: %s" % (server_and_media_id) ), ) # Test that the file is deleted self.assertFalse(os.path.exists(local_path))
class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): servlets = [ synapse.rest.admin.register_servlets, synapse.rest.admin.register_servlets_for_media_repo, login.register_servlets, profile.register_servlets, room.register_servlets, ] def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.media_repo = hs.get_media_repository_resource() self.server_name = hs.hostname self.admin_user = self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") self.filepaths = MediaFilePaths(hs.config.media.media_store_path) self.url = "/_synapse/admin/v1/media/%s/delete" % self.server_name # Move clock up to somewhat realistic time self.reactor.advance(1000000000) def test_no_auth(self) -> None: """ Try to delete media without authentication. """ channel = self.make_request("POST", self.url, b"{}") self.assertEqual( HTTPStatus.UNAUTHORIZED, channel.code, msg=channel.json_body, ) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) def test_requester_is_no_admin(self) -> None: """ If the user is not a server admin, an error is returned. """ self.other_user = self.register_user("user", "pass") self.other_user_token = self.login("user", "pass") channel = self.make_request( "POST", self.url, access_token=self.other_user_token, ) self.assertEqual( HTTPStatus.FORBIDDEN, channel.code, msg=channel.json_body, ) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) def test_media_is_not_local(self) -> None: """ Tests that a lookup for media that is not local returns a HTTPStatus.BAD_REQUEST """ url = "/_synapse/admin/v1/media/%s/delete" % "unknown_domain" channel = self.make_request( "POST", url + f"?before_ts={VALID_TIMESTAMP}", access_token=self.admin_user_tok, ) self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, msg=channel.json_body) self.assertEqual("Can only delete local media", channel.json_body["error"]) def test_missing_parameter(self) -> None: """ If the parameter `before_ts` is missing, an error is returned. """ channel = self.make_request( "POST", self.url, access_token=self.admin_user_tok, ) self.assertEqual( HTTPStatus.BAD_REQUEST, channel.code, msg=channel.json_body, ) self.assertEqual(Codes.MISSING_PARAM, channel.json_body["errcode"]) self.assertEqual("Missing integer query parameter 'before_ts'", channel.json_body["error"]) def test_invalid_parameter(self) -> None: """ If parameters are invalid, an error is returned. """ channel = self.make_request( "POST", self.url + "?before_ts=-1234", access_token=self.admin_user_tok, ) self.assertEqual( HTTPStatus.BAD_REQUEST, channel.code, msg=channel.json_body, ) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) self.assertEqual( "Query parameter before_ts must be a positive integer.", channel.json_body["error"], ) channel = self.make_request( "POST", self.url + f"?before_ts={INVALID_TIMESTAMP_IN_S}", access_token=self.admin_user_tok, ) self.assertEqual( HTTPStatus.BAD_REQUEST, channel.code, msg=channel.json_body, ) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) self.assertEqual( "Query parameter before_ts you provided is from the year 1970. " + "Double check that you are providing a timestamp in milliseconds.", channel.json_body["error"], ) channel = self.make_request( "POST", self.url + f"?before_ts={VALID_TIMESTAMP}&size_gt=-1234", access_token=self.admin_user_tok, ) self.assertEqual( HTTPStatus.BAD_REQUEST, channel.code, msg=channel.json_body, ) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) self.assertEqual( "Query parameter size_gt must be a string representing a positive integer.", channel.json_body["error"], ) channel = self.make_request( "POST", self.url + f"?before_ts={VALID_TIMESTAMP}&keep_profiles=not_bool", access_token=self.admin_user_tok, ) self.assertEqual( HTTPStatus.BAD_REQUEST, channel.code, msg=channel.json_body, ) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) self.assertEqual( "Boolean query parameter 'keep_profiles' must be one of ['true', 'false']", channel.json_body["error"], ) def test_delete_media_never_accessed(self) -> None: """ Tests that media deleted if it is older than `before_ts` and never accessed `last_access_ts` is `NULL` and `created_ts` < `before_ts` """ # upload and do not access server_and_media_id = self._create_media() self.pump(1.0) # test that the file exists media_id = server_and_media_id.split("/")[1] local_path = self.filepaths.local_media_filepath(media_id) self.assertTrue(os.path.exists(local_path)) # timestamp after upload/create now_ms = self.clock.time_msec() channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms), access_token=self.admin_user_tok, ) self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( media_id, channel.json_body["deleted_media"][0], ) self._access_media(server_and_media_id, False) def test_keep_media_by_date(self) -> None: """ Tests that media is not deleted if it is newer than `before_ts` """ # timestamp before upload now_ms = self.clock.time_msec() server_and_media_id = self._create_media() self._access_media(server_and_media_id) channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms), access_token=self.admin_user_tok, ) self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) self._access_media(server_and_media_id) # timestamp after upload now_ms = self.clock.time_msec() channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms), access_token=self.admin_user_tok, ) self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( server_and_media_id.split("/")[1], channel.json_body["deleted_media"][0], ) self._access_media(server_and_media_id, False) def test_keep_media_by_size(self) -> None: """ Tests that media is not deleted if its size is smaller than or equal to `size_gt` """ server_and_media_id = self._create_media() self._access_media(server_and_media_id) now_ms = self.clock.time_msec() channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms) + "&size_gt=67", access_token=self.admin_user_tok, ) self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) self._access_media(server_and_media_id) now_ms = self.clock.time_msec() channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms) + "&size_gt=66", access_token=self.admin_user_tok, ) self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( server_and_media_id.split("/")[1], channel.json_body["deleted_media"][0], ) self._access_media(server_and_media_id, False) def test_keep_media_by_user_avatar(self) -> None: """ Tests that we do not delete media if is used as a user avatar Tests parameter `keep_profiles` """ server_and_media_id = self._create_media() self._access_media(server_and_media_id) # set media as avatar channel = self.make_request( "PUT", "/profile/%s/avatar_url" % (self.admin_user, ), content={"avatar_url": "mxc://%s" % (server_and_media_id, )}, access_token=self.admin_user_tok, ) self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) now_ms = self.clock.time_msec() channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=true", access_token=self.admin_user_tok, ) self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) self._access_media(server_and_media_id) now_ms = self.clock.time_msec() channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=false", access_token=self.admin_user_tok, ) self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( server_and_media_id.split("/")[1], channel.json_body["deleted_media"][0], ) self._access_media(server_and_media_id, False) def test_keep_media_by_room_avatar(self) -> None: """ Tests that we do not delete media if it is used as a room avatar Tests parameter `keep_profiles` """ server_and_media_id = self._create_media() self._access_media(server_and_media_id) # set media as room avatar room_id = self.helper.create_room_as(self.admin_user, tok=self.admin_user_tok) channel = self.make_request( "PUT", "/rooms/%s/state/m.room.avatar" % (room_id, ), content={"url": "mxc://%s" % (server_and_media_id, )}, access_token=self.admin_user_tok, ) self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) now_ms = self.clock.time_msec() channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=true", access_token=self.admin_user_tok, ) self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) self._access_media(server_and_media_id) now_ms = self.clock.time_msec() channel = self.make_request( "POST", self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=false", access_token=self.admin_user_tok, ) self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( server_and_media_id.split("/")[1], channel.json_body["deleted_media"][0], ) self._access_media(server_and_media_id, False) def _create_media(self) -> str: """ Create a media and return media_id and server_and_media_id """ upload_resource = self.media_repo.children[b"upload"] # Upload some media into the room response = self.helper.upload_media( upload_resource, SMALL_PNG, tok=self.admin_user_tok, expect_code=HTTPStatus.OK, ) # Extract media ID from the response server_and_media_id = response["content_uri"][6:] # Cut off 'mxc://' server_name = server_and_media_id.split("/")[0] # Check that new media is a local and not remote self.assertEqual(server_name, self.server_name) return server_and_media_id def _access_media(self, server_and_media_id: str, expect_success: bool = True) -> None: """ Try to access a media and check the result """ download_resource = self.media_repo.children[b"download"] media_id = server_and_media_id.split("/")[1] local_path = self.filepaths.local_media_filepath(media_id) channel = make_request( self.reactor, FakeSite(download_resource, self.reactor), "GET", server_and_media_id, shorthand=False, access_token=self.admin_user_tok, ) if expect_success: self.assertEqual( HTTPStatus.OK, channel.code, msg=( "Expected to receive a HTTPStatus.OK on accessing media: %s" % server_and_media_id), ) # Test that the file exists self.assertTrue(os.path.exists(local_path)) else: self.assertEqual( HTTPStatus.NOT_FOUND, channel.code, msg= ("Expected to receive a HTTPStatus.NOT_FOUND on accessing deleted media: %s" % (server_and_media_id)), ) # Test that the file is deleted self.assertFalse(os.path.exists(local_path))