def test_archive_message_tool(self) -> None:
"""End-to-end test of the archiving tool, directly calling
archive_messages."""
# Make some expired messages in MIT:
expired_mit_msg_ids = self._make_mit_messages(
5,
timezone_now() - timedelta(days=MIT_REALM_DAYS+1)
)
# Make some non-expired messages in MIT:
self._make_mit_messages(4, timezone_now() - timedelta(days=MIT_REALM_DAYS-1))
# Change some Zulip messages to be expired:
expired_zulip_msg_ids = self._make_expired_zulip_messages(7)
expired_crossrealm_msg_id = self._send_cross_realm_personal_message()
# Make the message expired in the recipient's realm:
self._change_messages_date_sent(
[expired_crossrealm_msg_id],
timezone_now() - timedelta(ZULIP_REALM_DAYS+1)
)
expired_msg_ids = expired_mit_msg_ids + expired_zulip_msg_ids + [expired_crossrealm_msg_id]
expired_usermsg_ids = self._get_usermessage_ids(expired_msg_ids)
archive_messages(chunk_size=2) # Specify low chunk_size to test batching.
# Make sure we archived what needed:
self._verify_archive_data(expired_msg_ids, expired_usermsg_ids)
restore_all_data_from_archive()
self._verify_restored_data(expired_msg_ids, expired_usermsg_ids)
def test_clean_archived_data(self) -> None:
self._make_expired_zulip_messages(7)
archive_messages(
chunk_size=2) # Small chunk size to have multiple transactions
transactions = list(ArchiveTransaction.objects.all())
for transaction in transactions[0:-1]:
transaction.timestamp = timezone_now() - timedelta(
days=settings.ARCHIVED_DATA_VACUUMING_DELAY_DAYS + 1)
transaction.save()
message_ids_to_clean = list(
ArchivedMessage.objects.filter(
archive_transaction__in=transactions[0:-1]).values_list(
"id", flat=True))
clean_archived_data()
remaining_transactions = list(ArchiveTransaction.objects.all())
self.assert_length(remaining_transactions, 1)
# All transactions except the last one were deleted:
self.assertEqual(remaining_transactions[0].id, transactions[-1].id)
# And corresponding ArchivedMessages should have been deleted:
self.assertFalse(
ArchivedMessage.objects.filter(
id__in=message_ids_to_clean).exists())
self.assertFalse(
ArchivedUserMessage.objects.filter(
message_id__in=message_ids_to_clean).exists())
for message in ArchivedMessage.objects.all():
self.assertEqual(message.archive_transaction_id,
remaining_transactions[0].id)
def test_archive_message_tool(self) -> None:
"""End-to-end test of the archiving tool, directly calling
archive_messages."""
# Make some expired messages in MIT:
expired_mit_msg_ids = self._make_mit_messages(
5,
timezone_now() - timedelta(days=MIT_REALM_DAYS + 1))
# Make some non-expired messages in MIT:
self._make_mit_messages(
4,
timezone_now() - timedelta(days=MIT_REALM_DAYS - 1))
# Change some Zulip messages to be expired:
expired_zulip_msg_ids = self._make_expired_zulip_messages(7)
expired_crossrealm_msg_id = self._send_cross_realm_message()
# Make the message expired on both realms:
self._change_messages_pub_date([expired_crossrealm_msg_id],
timezone_now() -
timedelta(MIT_REALM_DAYS + 1))
expired_msg_ids = expired_mit_msg_ids + expired_zulip_msg_ids
expired_usermsg_ids = self._get_usermessage_ids(expired_msg_ids)
archive_messages()
# Make sure we archived what neeeded:
self._verify_archive_data(expired_msg_ids, expired_usermsg_ids)
# Check that the archived messages were deleted:
self.assertEqual(
UserMessage.objects.filter(id__in=expired_usermsg_ids).count(), 0)
self.assertEqual(
Message.objects.filter(id__in=expired_msg_ids).count(), 0)
def test_expired_messages_in_each_realm(self) -> None:
"""General test for archiving expired messages properly with
multiple realms involved"""
# Make some expired messages in MIT:
expired_mit_msg_ids = self._make_mit_messages(
5,
timezone_now() - timedelta(days=MIT_REALM_DAYS + 1),
)
# Make some non-expired messages in MIT:
self._make_mit_messages(
4,
timezone_now() - timedelta(days=MIT_REALM_DAYS - 1))
# Change some Zulip messages to be expired:
expired_zulip_msg_ids = list(
Message.objects.order_by("id").filter(
sender__realm=self.zulip_realm).values_list("id",
flat=True)[3:10])
self._change_messages_date_sent(
expired_zulip_msg_ids,
timezone_now() - timedelta(ZULIP_REALM_DAYS + 1),
)
expired_msg_ids = expired_mit_msg_ids + expired_zulip_msg_ids
expired_usermsg_ids = self._get_usermessage_ids(expired_msg_ids)
archive_messages()
self._verify_archive_data(expired_msg_ids, expired_usermsg_ids)
restore_all_data_from_archive()
self._verify_restored_data(expired_msg_ids, expired_usermsg_ids)
def test_archiving_interrupted(self) -> None:
"""Check that queries get rolled back to a consistent state
if archiving gets interrupted in the middle of processing a chunk."""
expired_msg_ids = self._make_expired_zulip_messages(7)
expired_usermsg_ids = self._get_usermessage_ids(expired_msg_ids)
# Insert an exception near the end of the archiving process of a chunk:
with mock.patch("zerver.lib.retention.delete_messages",
side_effect=Exception):
with self.assertRaises(Exception):
# Specify large chunk_size to ensure things happen in a single batch
archive_messages(chunk_size=1000)
# Archiving code has been executed, but because we got an exception, things should have been rolled back:
self._verify_archive_data([], [])
self.assertEqual(
set(
Message.objects.filter(id__in=expired_msg_ids).values_list(
"id", flat=True)),
set(expired_msg_ids),
)
self.assertEqual(
set(
UserMessage.objects.filter(
id__in=expired_usermsg_ids).values_list("id",
flat=True)),
set(expired_usermsg_ids),
)
def test_restoring_and_rearchiving(self) -> None:
expired_msg_ids = self._make_mit_messages(
7,
timezone_now() - timedelta(days=MIT_REALM_DAYS + 1),
)
expired_usermsg_ids = self._get_usermessage_ids(expired_msg_ids)
archive_messages(chunk_size=4)
self._verify_archive_data(expired_msg_ids, expired_usermsg_ids)
transactions = ArchiveTransaction.objects.all()
self.assert_length(
transactions,
2) # With chunk_size 4, there should be 2 transactions
restore_all_data_from_archive()
transactions[0].refresh_from_db()
transactions[1].refresh_from_db()
self.assertTrue(transactions[0].restored)
self.assertTrue(transactions[1].restored)
archive_messages(chunk_size=10)
self._verify_archive_data(expired_msg_ids, expired_usermsg_ids)
transactions = ArchiveTransaction.objects.order_by("id")
self.assert_length(transactions, 3)
archived_messages = ArchivedMessage.objects.filter(
id__in=expired_msg_ids)
# Check that the re-archived messages are correctly assigned to the new transaction:
for message in archived_messages:
self.assertEqual(message.archive_transaction_id,
transactions[2].id)
def test_expired_messages_in_one_realm(self) -> None:
"""Test with a retention policy set for only the MIT realm"""
self._set_realm_message_retention_value(self.zulip_realm, None)
# Make some expired messages in MIT:
expired_mit_msg_ids = self._make_mit_messages(
5,
timezone_now() - timedelta(days=MIT_REALM_DAYS+1)
)
# Make some non-expired messages in MIT:
self._make_mit_messages(4, timezone_now() - timedelta(days=MIT_REALM_DAYS-1))
# Change some Zulip messages date_sent, but the realm has no retention policy,
# so they shouldn't get archived
zulip_msg_ids = list(Message.objects.order_by('id').filter(
sender__realm=self.zulip_realm).values_list('id', flat=True)[3:10])
self._change_messages_date_sent(
zulip_msg_ids,
timezone_now() - timedelta(ZULIP_REALM_DAYS+1)
)
# Only MIT has a retention policy:
expired_msg_ids = expired_mit_msg_ids
expired_usermsg_ids = self._get_usermessage_ids(expired_msg_ids)
archive_messages()
self._verify_archive_data(expired_msg_ids, expired_usermsg_ids)
restore_all_data_from_archive()
self._verify_restored_data(expired_msg_ids, expired_usermsg_ids)
self._set_realm_message_retention_value(self.zulip_realm, ZULIP_REALM_DAYS)
def test_archiving_reactions(self) -> None:
expired_msg_ids = self._make_expired_zulip_messages(2)
self.post_zulip_reaction(expired_msg_ids[0], 'hamlet')
self.post_zulip_reaction(expired_msg_ids[0], 'cordelia')
self.post_zulip_reaction(expired_msg_ids[1], 'hamlet')
reaction_ids = list(
Reaction.objects.filter(message_id__in=expired_msg_ids).values_list('id', flat=True)
)
self.assertEqual(len(reaction_ids), 3)
self.assertEqual(Reaction.objects.filter(id__in=reaction_ids).count(), 3)
archive_messages()
self.assertEqual(Reaction.objects.filter(id__in=reaction_ids).count(), 0)
self.assertEqual(
set(ArchivedReaction.objects.filter(id__in=reaction_ids).values_list('id', flat=True)),
set(reaction_ids)
)
restore_all_data_from_archive()
self.assertEqual(
set(Reaction.objects.filter(id__in=reaction_ids).values_list('id', flat=True)),
set(reaction_ids)
)
def test_archive_message_tool(self) -> None:
"""End-to-end test of the archiving tool, directly calling
archive_messages."""
expected_message_ids_dict = self._make_expired_messages()
# We also include a cross-realm message in this test.
sent_cross_realm_message_id = self._send_cross_realm_message()
expected_message_ids_dict['mit_msgs_ids'].append(
sent_cross_realm_message_id)
self._change_messages_pub_date([sent_cross_realm_message_id],
timezone_now() -
timedelta(days=MIT_REALM_DAYS + 1))
expected_message_ids = expected_message_ids_dict[
'mit_msgs_ids'] + expected_message_ids_dict['zulip_msgs_ids']
# Get expired user messages by message ids
expected_user_msgs_ids = list(
UserMessage.objects.filter(message_id__in=expected_message_ids).
order_by('id').values_list('id', flat=True))
msgs_qty = Message.objects.count()
archive_messages()
# Temporary debugging code while we investigate CI failures
print(expected_message_ids)
print(ArchivedMessage.objects.all())
# Compare archived messages and user messages with expired messages
self.assertEqual(ArchivedMessage.objects.count(),
len(expected_message_ids))
self.assertEqual(ArchivedUserMessage.objects.count(),
len(expected_user_msgs_ids))
# Check non-archived messages messages after removing expired
# messages from main tables without cross-realm messages.
self.assertEqual(Message.objects.count(),
msgs_qty - ArchivedMessage.objects.count())
self.assertEqual(
Message.objects.filter(
id__in=expected_message_ids_dict['zulip_msgs_ids']).count(), 0)
self.assertEqual(
Message.objects.filter(
id__in=expected_message_ids_dict['mit_msgs_ids']).count(), 0)
self.assertEqual(
Message.objects.filter(
id__in=expected_message_ids_dict['zulip_msgs_ids']).count(), 0)
# Check archived messages by realm using our standard checker
# function; we add the cross-realm message ID to the
# zulip_realm list for this test because its sender lives in
# that realm in the development environment.
expected_message_ids_dict['zulip_msgs_ids'].append(
sent_cross_realm_message_id)
self._check_archived_messages_ids_by_realm(
expected_message_ids_dict['zulip_msgs_ids'], self.zulip_realm)
self._check_archived_messages_ids_by_realm(
expected_message_ids_dict['mit_msgs_ids'], self.mit_realm)
def test_cross_realm_personal_message_archiving(self) -> None:
"""Check that cross-realm personal messages get correctly archived. """
msg_ids = [self._send_cross_realm_personal_message() for i in range(1, 7)]
usermsg_ids = self._get_usermessage_ids(msg_ids)
# Make the message expired on the recipient's realm:
self._change_messages_date_sent(msg_ids, timezone_now() - timedelta(ZULIP_REALM_DAYS+1))
archive_messages()
self._verify_archive_data(msg_ids, usermsg_ids)
def test_archiving_submessages(self) -> None:
expired_msg_ids = self._make_expired_zulip_messages(2)
cordelia = self.example_user("cordelia")
hamlet = self.example_user("hamlet")
do_add_submessage(
realm=self.zulip_realm,
sender_id=cordelia.id,
message_id=expired_msg_ids[0],
msg_type="whatever",
content='{"name": "alice", "salary": 20}',
)
do_add_submessage(
realm=self.zulip_realm,
sender_id=hamlet.id,
message_id=expired_msg_ids[0],
msg_type="whatever",
content='{"name": "john", "salary": 30}',
)
do_add_submessage(
realm=self.zulip_realm,
sender_id=cordelia.id,
message_id=expired_msg_ids[1],
msg_type="whatever",
content='{"name": "jack", "salary": 10}',
)
submessage_ids = list(
SubMessage.objects.filter(
message_id__in=expired_msg_ids).values_list("id", flat=True), )
self.assert_length(submessage_ids, 3)
self.assertEqual(
SubMessage.objects.filter(id__in=submessage_ids).count(), 3)
archive_messages()
self.assertEqual(
SubMessage.objects.filter(id__in=submessage_ids).count(), 0)
self.assertEqual(
set(
ArchivedSubMessage.objects.filter(
id__in=submessage_ids).values_list("id", flat=True)),
set(submessage_ids),
)
restore_all_data_from_archive()
self.assertEqual(
set(
SubMessage.objects.filter(id__in=submessage_ids).values_list(
"id", flat=True)),
set(submessage_ids),
)
def test_archive_message_tool(self) -> None:
"""End-to-end test of the archiving tool, directly calling
archive_messages."""
# Make some expired messages in MIT:
expired_mit_msg_ids = self._make_mit_messages(
5,
timezone_now() - timedelta(days=MIT_REALM_DAYS + 1))
# Make some non-expired messages in MIT:
self._make_mit_messages(
4,
timezone_now() - timedelta(days=MIT_REALM_DAYS - 1))
# Change some Zulip messages to be expired:
expired_zulip_msg_ids = self._make_expired_zulip_messages(7)
expired_crossrealm_msg_id = self._send_cross_realm_message()
# Make the message expired on both realms:
self._change_messages_pub_date([expired_crossrealm_msg_id],
timezone_now() -
timedelta(MIT_REALM_DAYS + 1))
expired_msg_ids = expired_mit_msg_ids + expired_zulip_msg_ids + [
expired_crossrealm_msg_id
]
# We explicitly call list() because we need to force evaluation of the query, before the
# UserMessage objects get deleted from the database by archive_messages():
expired_usermsg_ids = list(
UserMessage.objects.filter(
message_id__in=expired_msg_ids).values_list('id', flat=True))
archive_messages()
# Make sure we archived what neeeded:
self.assertEqual(
set(ArchivedMessage.objects.values_list('id', flat=True)),
set(expired_msg_ids))
self.assertEqual(
set(ArchivedUserMessage.objects.values_list('id', flat=True)),
set(expired_usermsg_ids))
# Check that the archived messages were deleted:
# But the cross-realm message's recipient's usermessage will not be deleted
# in the current implementation:
self.assertEqual(
UserMessage.objects.get(id__in=expired_usermsg_ids).message_id,
expired_crossrealm_msg_id)
self.assertEqual(
UserMessage.objects.get(
id__in=expired_usermsg_ids).user_profile.realm_id,
self.mit_realm.id)
self.assertEqual(
Message.objects.get(id__in=expired_msg_ids).id,
expired_crossrealm_msg_id)
def test_archive_message_tool(self) -> None:
"""End-to-end test of the archiving tool, directly calling
archive_messages."""
# Make some expired messages in MIT:
expired_mit_msg_ids = self._make_mit_messages(
5,
timezone_now() - timedelta(days=MIT_REALM_DAYS + 1))
# Make some non-expired messages in MIT:
self._make_mit_messages(
4,
timezone_now() - timedelta(days=MIT_REALM_DAYS - 1))
# Change some Zulip messages to be expired:
expired_zulip_msg_ids = list(
Message.objects.order_by('id').filter(
sender__realm=self.zulip_realm).values_list('id',
flat=True)[3:10])
self._change_messages_pub_date(
expired_zulip_msg_ids,
timezone_now() - timedelta(ZULIP_REALM_DAYS + 1))
expired_crossrealm_msg_id = self._send_cross_realm_message()
# Make the message expired on both realms:
self._change_messages_pub_date([expired_crossrealm_msg_id],
timezone_now() -
timedelta(MIT_REALM_DAYS + 1))
expired_msg_ids = expired_mit_msg_ids + expired_zulip_msg_ids + [
expired_crossrealm_msg_id
]
# We explicitly call list() because we need to force evaluation of the query, before the
# UserMessage objects get deleted from the database by archive_messages():
expired_usermsg_ids = list(
UserMessage.objects.filter(
message_id__in=expired_msg_ids).values_list('id', flat=True))
archive_messages()
# Make sure we archived what neeeded:
self.assertEqual(
set(ArchivedMessage.objects.values_list('id', flat=True)),
set(expired_msg_ids))
self.assertEqual(
set(ArchivedUserMessage.objects.values_list('id', flat=True)),
set(expired_usermsg_ids))
# Check that the archived messages were deleted:
self.assertFalse(
Message.objects.filter(id__in=expired_msg_ids).exists())
self.assertFalse(
UserMessage.objects.filter(id__in=expired_usermsg_ids).exists())
def test_cross_realm_messages_not_archived(self) -> None:
"""Check that cross-realm messages don't get archived or deleted."""
msg_id = self._send_cross_realm_message()
# Make the message expired on both realms:
self._change_messages_pub_date([msg_id],
timezone_now() -
timedelta(MIT_REALM_DAYS + 1))
archive_messages()
self.assertEqual(ArchivedMessage.objects.count(), 0)
self.assertEqual(ArchivedUserMessage.objects.count(), 0)
self.assertEqual(
UserMessage.objects.filter(message_id=msg_id).count(), 2)
self.assertTrue(Message.objects.filter(id=msg_id).exists())
def test_restore_retention_policy_deletions_for_stream(self) -> None:
cordelia = self.example_user("cordelia")
hamlet = self.example_user("hamlet")
realm = get_realm("zulip")
stream_name = "Verona"
stream = get_stream(stream_name, realm)
message_ids_to_archive_manually = [
self.send_stream_message(cordelia, stream_name, str(i))
for i in range(0, 2)
]
usermessage_ids_to_archive_manually = self._get_usermessage_ids(
message_ids_to_archive_manually)
message_ids_to_archive_by_policy = [
self.send_stream_message(hamlet, stream_name, str(i))
for i in range(0, 2)
]
usermessage_ids_to_archive_by_policy = self._get_usermessage_ids(
message_ids_to_archive_by_policy)
expected_archived_message_ids = (message_ids_to_archive_manually +
message_ids_to_archive_by_policy)
expected_archived_usermessage_ids = (
usermessage_ids_to_archive_manually +
usermessage_ids_to_archive_by_policy)
self._set_stream_message_retention_value(stream, 5)
self._change_messages_date_sent(message_ids_to_archive_by_policy,
timezone_now() - timedelta(days=6))
move_messages_to_archive(message_ids_to_archive_manually)
archive_messages()
self._verify_archive_data(expected_archived_message_ids,
expected_archived_usermessage_ids)
restore_retention_policy_deletions_for_stream(stream)
# Verify that we restore the stream messages that were archived due to retention policy,
# but not the ones manually deleted.
self.assert_length(
Message.objects.filter(id__in=message_ids_to_archive_by_policy),
len(message_ids_to_archive_by_policy),
)
self.assertFalse(
Message.objects.filter(id__in=message_ids_to_archive_manually))
def test_archiving_reactions(self) -> None:
expired_msg_ids = self._make_expired_zulip_messages(2)
hamlet = self.example_user("hamlet")
cordelia = self.example_user("cordelia")
for sender in [hamlet, cordelia]:
self.api_post(
sender,
f"/api/v1/messages/{expired_msg_ids[0]}/reactions",
zulip_reaction_info(),
)
self.api_post(
hamlet,
f"/api/v1/messages/{expired_msg_ids[1]}/reactions",
zulip_reaction_info(),
)
reaction_ids = list(
Reaction.objects.filter(
message_id__in=expired_msg_ids).values_list("id", flat=True), )
self.assert_length(reaction_ids, 3)
self.assertEqual(
Reaction.objects.filter(id__in=reaction_ids).count(), 3)
archive_messages()
self.assertEqual(
Reaction.objects.filter(id__in=reaction_ids).count(), 0)
self.assertEqual(
set(
ArchivedReaction.objects.filter(
id__in=reaction_ids).values_list("id", flat=True)),
set(reaction_ids),
)
restore_all_data_from_archive()
self.assertEqual(
set(
Reaction.objects.filter(id__in=reaction_ids).values_list(
"id", flat=True)),
set(reaction_ids),
)
def test_different_stream_realm_policies(self) -> None:
verona = get_stream("Verona", self.zulip_realm)
hamlet = self.example_user("hamlet")
msg_id = self.send_stream_message(hamlet, "Verona", "test")
usermsg_ids = self._get_usermessage_ids([msg_id])
self._change_messages_date_sent([msg_id],
timezone_now() - timedelta(days=2))
# Don't archive if stream's retention policy set to -1:
self._set_realm_message_retention_value(self.zulip_realm, 1)
self._set_stream_message_retention_value(verona, -1)
archive_messages()
self._verify_archive_data([], [])
# Don't archive if stream and realm have no retention policy:
self._set_realm_message_retention_value(self.zulip_realm, -1)
self._set_stream_message_retention_value(verona, None)
archive_messages()
self._verify_archive_data([], [])
# Archive if stream has a retention policy set:
self._set_realm_message_retention_value(self.zulip_realm, -1)
self._set_stream_message_retention_value(verona, 1)
archive_messages()
self._verify_archive_data([msg_id], usermsg_ids)
def test_archiving_submessages(self) -> None:
expired_msg_ids = self._make_expired_zulip_messages(2)
cordelia = self.example_user('cordelia')
hamlet = self.example_user('hamlet')
do_add_submessage(realm=self.zulip_realm,
sender_id=cordelia.id,
message_id=expired_msg_ids[0],
msg_type='whatever',
content='{"name": "alice", "salary": 20}')
do_add_submessage(realm=self.zulip_realm,
sender_id=hamlet.id,
message_id=expired_msg_ids[0],
msg_type='whatever',
content='{"name": "john", "salary": 30}')
do_add_submessage(realm=self.zulip_realm,
sender_id=cordelia.id,
message_id=expired_msg_ids[1],
msg_type='whatever',
content='{"name": "jack", "salary": 10}')
submessage_ids = list(
SubMessage.objects.filter(
message_id__in=expired_msg_ids).values_list('id', flat=True))
self.assertEqual(len(submessage_ids), 3)
self.assertEqual(
SubMessage.objects.filter(id__in=submessage_ids).count(), 3)
archive_messages()
self.assertEqual(
SubMessage.objects.filter(id__in=submessage_ids).count(), 0)
self.assertEqual(
set(
ArchivedSubMessage.objects.filter(
id__in=submessage_ids).values_list('id', flat=True)),
set(submessage_ids))
def test_archiving_attachments(self) -> None:
"""End-to-end test for the logic for archiving attachments. This test
is hard to read without first reading _send_messages_with_attachments"""
msgs_ids = self._send_messages_with_attachments()
# First, confirm deleting the oldest message
# (`expired_message_id`) creates ArchivedAttachment objects
# and associates that message ID with them, but does not
# delete the Attachment object.
archive_messages()
self.assertEqual(ArchivedAttachment.objects.count(), 3)
self.assertEqual(
list(
ArchivedAttachment.objects.distinct(
"messages__id").values_list("messages__id", flat=True)),
[msgs_ids["expired_message_id"]],
)
self.assertEqual(Attachment.objects.count(), 3)
# Now make `actual_message_id` expired too. We still don't
# delete the Attachment objects.
self._change_messages_date_sent([msgs_ids["actual_message_id"]],
timezone_now() -
timedelta(days=MIT_REALM_DAYS + 1))
archive_messages()
self.assertEqual(Attachment.objects.count(), 3)
# Finally, make the last message mentioning those attachments
# expired. We should now delete the Attachment objects and
# each ArchivedAttachment object should list all 3 messages.
self._change_messages_date_sent([msgs_ids["other_user_message_id"]],
timezone_now() -
timedelta(days=MIT_REALM_DAYS + 1))
archive_messages()
self.assertEqual(Attachment.objects.count(), 0)
self.assertEqual(ArchivedAttachment.objects.count(), 3)
self.assertEqual(
list(
ArchivedAttachment.objects.distinct("messages__id").order_by(
"messages__id").values_list("messages__id", flat=True)),
sorted(msgs_ids.values()),
)
restore_all_data_from_archive()
# Attachments should have been restored:
self.assertEqual(Attachment.objects.count(), 3)
# Archived data doesn't get deleted by restoring.
self.assertEqual(ArchivedAttachment.objects.count(), 3)
self.assertEqual(
list(
Attachment.objects.distinct("messages__id").order_by(
"messages__id").values_list("messages__id", flat=True)),
sorted(msgs_ids.values()),
)
def test_archiving_attachments(self) -> None:
"""End-to-end test for the logic for archiving attachments. This test
is hard to read without first reading _send_messages_with_attachments"""
msgs_ids = self._send_messages_with_attachments()
# First, confirm deleting the oldest message
# (`expired_message_id`) creates ArchivedAttachment objects
# and associates that message ID with them, but does not
# delete the Attachment object.
archive_messages()
archived_attachment = ArchivedAttachment.objects.all()
attachment = Attachment.objects.all()
self.assertEqual(archived_attachment.count(), 3)
self.assertEqual(
list(
archived_attachment.distinct('messages__id').values_list(
'messages__id', flat=True)),
[msgs_ids['expired_message_id']])
self.assertEqual(attachment.count(), 3)
# Now make `actual_message_id` expired too. We still don't
# delete the Attachment objects.
self._change_messages_pub_date([msgs_ids['actual_message_id']],
timezone_now() -
timedelta(days=MIT_REALM_DAYS + 1))
archive_messages()
self.assertEqual(attachment.count(), 3)
# Finally, make the last message mentioning those attachments
# expired. We should now delete the Attachment objects and
# each ArchivedAttachment object should list all 3 messages.
self._change_messages_pub_date([msgs_ids['other_user_message_id']],
timezone_now() -
timedelta(days=MIT_REALM_DAYS + 1))
archive_messages()
self.assertEqual(attachment.count(), 0)
self.assertEqual(archived_attachment.count(), 3)
self.assertEqual(
list(
archived_attachment.distinct('messages__id').order_by(
'messages__id').values_list('messages__id', flat=True)),
sorted(msgs_ids.values()))
def handle(self, *args: Any, **options: str) -> None:
clean_archived_data()
archive_messages()
def test_no_expired_messages(self) -> None:
archive_messages()
self.assertEqual(ArchivedUserMessage.objects.count(), 0)
self.assertEqual(ArchivedMessage.objects.count(), 0)
def handle(self, *args: Any, **options: str) -> None:
archive_messages()
