def test_endpoint_local_uploads(self) -> None:
admin = self.example_user('iago')
self.login_user(admin)
tarball_path = create_dummy_file('test-export.tar.gz')
# Test the export logic.
with patch('zerver.lib.export.do_export_realm',
return_value=tarball_path) as mock_export:
with stdout_suppressed():
result = self.client_post('/json/export/realm')
self.assert_json_success(result)
self.assertFalse(os.path.exists(tarball_path))
args = mock_export.call_args_list[0][1]
self.assertEqual(args['realm'], admin.realm)
self.assertEqual(args['public_only'], True)
self.assertIn('/tmp/zulip-export-', args['output_dir'])
self.assertEqual(args['threads'], 6)
# Get the entry and test that iago initiated it.
audit_log_entry = RealmAuditLog.objects.filter(
event_type=RealmAuditLog.REALM_EXPORTED).first()
self.assertEqual(audit_log_entry.acting_user_id, admin.id)
# Test that the file is hosted, and the contents are as expected.
path_id = ujson.loads(audit_log_entry.extra_data).get('export_path')
response = self.client_get(path_id)
self.assertEqual(response.status_code, 200)
self.assert_url_serves_contents_of_file(path_id, b'zulip!')
result = self.client_get('/json/export/realm')
self.assert_json_success(result)
# Test that the export we have is the export we created.
export_dict = result.json()['exports']
self.assertEqual(export_dict[0]['id'], audit_log_entry.id)
self.assertEqual(export_dict[0]['export_url'],
admin.realm.uri + path_id)
self.assertEqual(export_dict[0]['acting_user_id'], admin.id)
self.assert_length(
export_dict,
RealmAuditLog.objects.filter(
realm=admin.realm,
event_type=RealmAuditLog.REALM_EXPORTED).count())
# Finally, delete the file.
result = self.client_delete(f'/json/export/realm/{audit_log_entry.id}')
self.assert_json_success(result)
response = self.client_get(path_id)
self.assertEqual(response.status_code, 404)
# Try to delete an export with a `deleted_timestamp` key.
audit_log_entry.refresh_from_db()
export_data = ujson.loads(audit_log_entry.extra_data)
self.assertIn('deleted_timestamp', export_data)
result = self.client_delete(f'/json/export/realm/{audit_log_entry.id}')
self.assert_json_error(result, "Export already deleted")
# Now try to delete a non-existent export.
result = self.client_delete('/json/export/realm/0')
self.assert_json_error(result, "Invalid data export ID")
def test_endpoint_s3(self) -> None:
admin = self.example_user('iago')
self.login_user(admin)
bucket = create_s3_buckets(settings.S3_AVATAR_BUCKET)[0]
tarball_path = create_dummy_file('test-export.tar.gz')
# Test the export logic.
with patch('zerver.lib.export.do_export_realm',
return_value=tarball_path) as mock_export:
with self.settings(LOCAL_UPLOADS_DIR=None), stdout_suppressed():
result = self.client_post('/json/export/realm')
self.assert_json_success(result)
self.assertFalse(os.path.exists(tarball_path))
args = mock_export.call_args_list[0][1]
self.assertEqual(args['realm'], admin.realm)
self.assertEqual(args['public_only'], True)
self.assertIn('/tmp/zulip-export-', args['output_dir'])
self.assertEqual(args['threads'], 6)
# Get the entry and test that iago initiated it.
audit_log_entry = RealmAuditLog.objects.filter(
event_type=RealmAuditLog.REALM_EXPORTED).first()
self.assertEqual(audit_log_entry.acting_user_id, admin.id)
# Test that the file is hosted, and the contents are as expected.
path_id = ujson.loads(audit_log_entry.extra_data).get('export_path')
self.assertIsNotNone(path_id)
self.assertEqual(bucket.Object(path_id).get()['Body'].read(), b'zulip!')
result = self.client_get('/json/export/realm')
self.assert_json_success(result)
# Test that the export we have is the export we created.
export_dict = result.json()['exports']
self.assertEqual(export_dict[0]['id'], audit_log_entry.id)
self.assertEqual(export_dict[0]['export_url'],
'https://test-avatar-bucket.s3.amazonaws.com' + path_id)
self.assertEqual(export_dict[0]['acting_user_id'], admin.id)
self.assert_length(export_dict,
RealmAuditLog.objects.filter(
realm=admin.realm,
event_type=RealmAuditLog.REALM_EXPORTED).count())
# Finally, delete the file.
result = self.client_delete(f'/json/export/realm/{audit_log_entry.id}')
self.assert_json_success(result)
with self.assertRaises(botocore.exceptions.ClientError):
bucket.Object(path_id).load()
# Try to delete an export with a `deleted_timestamp` key.
audit_log_entry.refresh_from_db()
export_data = ujson.loads(audit_log_entry.extra_data)
self.assertIn('deleted_timestamp', export_data)
result = self.client_delete(f'/json/export/realm/{audit_log_entry.id}')
self.assert_json_error(result, "Export already deleted")
# Now try to delete a non-existent export.
result = self.client_delete('/json/export/realm/0')
self.assert_json_error(result, "Invalid data export ID")
def test_management_commands_show_help(self) -> None:
with stdout_suppressed():
for command in self.commands:
with self.subTest(management_command=command):
with self.assertRaises(SystemExit):
call_command(command, "--help")
# zerver/management/commands/runtornado.py sets this to True;
# we need to reset it here. See #3685 for details.
settings.RUNNING_INSIDE_TORNADO = False
def test_management_commands_show_help(self) -> None:
with stdout_suppressed() as stdout:
for command in self.commands:
print('Testing management command: {}'.format(command),
file=stdout)
with self.assertRaises(SystemExit):
call_command(command, '--help')
# zerver/management/commands/runtornado.py sets this to True;
# we need to reset it here. See #3685 for details.
settings.RUNNING_INSIDE_TORNADO = False
def test_management_commands_show_help(self) -> None:
with stdout_suppressed() as stdout:
for command in self.commands:
print('Testing management command: {}'.format(command),
file=stdout)
with self.assertRaises(SystemExit):
call_command(command, '--help')
# zerver/management/commands/runtornado.py sets this to True;
# we need to reset it here. See #3685 for details.
settings.RUNNING_INSIDE_TORNADO = False
def test_endpoint_s3(self) -> None:
admin = self.example_user("iago")
self.login_user(admin)
bucket = create_s3_buckets(settings.S3_AVATAR_BUCKET)[0]
tarball_path = create_dummy_file("test-export.tar.gz")
# Test the export logic.
with patch("zerver.lib.export.do_export_realm",
return_value=tarball_path) as mock_export:
with self.settings(LOCAL_UPLOADS_DIR=None), stdout_suppressed(
), self.assertLogs(level="INFO") as info_logs:
result = self.client_post("/json/export/realm")
self.assertTrue("INFO:root:Completed data export for zulip in " in
info_logs.output[0])
self.assert_json_success(result)
self.assertFalse(os.path.exists(tarball_path))
args = mock_export.call_args_list[0][1]
self.assertEqual(args["realm"], admin.realm)
self.assertEqual(args["public_only"], True)
self.assertIn("/tmp/zulip-export-", args["output_dir"])
self.assertEqual(args["threads"], 6)
# Get the entry and test that iago initiated it.
audit_log_entry = RealmAuditLog.objects.filter(
event_type=RealmAuditLog.REALM_EXPORTED).first()
assert audit_log_entry is not None
self.assertEqual(audit_log_entry.acting_user_id, admin.id)
# Test that the file is hosted, and the contents are as expected.
extra_data = audit_log_entry.extra_data
assert extra_data is not None
export_path = orjson.loads(extra_data)["export_path"]
assert export_path.startswith("/")
path_id = export_path[1:]
self.assertEqual(
bucket.Object(path_id).get()["Body"].read(), b"zulip!")
result = self.client_get("/json/export/realm")
self.assert_json_success(result)
# Test that the export we have is the export we created.
export_dict = result.json()["exports"]
self.assertEqual(export_dict[0]["id"], audit_log_entry.id)
self.assertEqual(
export_dict[0]["export_url"],
"https://test-avatar-bucket.s3.amazonaws.com" + export_path,
)
self.assertEqual(export_dict[0]["acting_user_id"], admin.id)
self.assert_length(
export_dict,
RealmAuditLog.objects.filter(
realm=admin.realm,
event_type=RealmAuditLog.REALM_EXPORTED).count(),
)
# Finally, delete the file.
result = self.client_delete(f"/json/export/realm/{audit_log_entry.id}")
self.assert_json_success(result)
with self.assertRaises(botocore.exceptions.ClientError):
bucket.Object(path_id).load()
# Try to delete an export with a `deleted_timestamp` key.
audit_log_entry.refresh_from_db()
extra_data = audit_log_entry.extra_data
assert extra_data is not None
export_data = orjson.loads(extra_data)
self.assertIn("deleted_timestamp", export_data)
result = self.client_delete(f"/json/export/realm/{audit_log_entry.id}")
self.assert_json_error(result, "Export already deleted")
# Now try to delete a non-existent export.
result = self.client_delete("/json/export/realm/0")
self.assert_json_error(result, "Invalid data export ID")
def test_endpoint_local_uploads(self) -> None:
admin = self.example_user("iago")
self.login_user(admin)
tarball_path = create_dummy_file("test-export.tar.gz")
# Test the export logic.
with patch("zerver.lib.export.do_export_realm",
return_value=tarball_path) as mock_export:
with stdout_suppressed(), self.assertLogs(
level="INFO") as info_logs:
result = self.client_post("/json/export/realm")
self.assertTrue("INFO:root:Completed data export for zulip in " in
info_logs.output[0])
self.assert_json_success(result)
self.assertFalse(os.path.exists(tarball_path))
args = mock_export.call_args_list[0][1]
self.assertEqual(args["realm"], admin.realm)
self.assertEqual(args["public_only"], True)
self.assertIn("/tmp/zulip-export-", args["output_dir"])
self.assertEqual(args["threads"], 6)
# Get the entry and test that iago initiated it.
audit_log_entry = RealmAuditLog.objects.filter(
event_type=RealmAuditLog.REALM_EXPORTED).first()
assert audit_log_entry is not None
self.assertEqual(audit_log_entry.acting_user_id, admin.id)
# Test that the file is hosted, and the contents are as expected.
extra_data = audit_log_entry.extra_data
assert extra_data is not None
export_path = orjson.loads(extra_data).get("export_path")
response = self.client_get(export_path)
self.assertEqual(response.status_code, 200)
self.assert_streaming_content(response, b"zulip!")
result = self.client_get("/json/export/realm")
self.assert_json_success(result)
# Test that the export we have is the export we created.
export_dict = result.json()["exports"]
self.assertEqual(export_dict[0]["id"], audit_log_entry.id)
self.assertEqual(export_dict[0]["export_url"],
admin.realm.uri + export_path)
self.assertEqual(export_dict[0]["acting_user_id"], admin.id)
self.assert_length(
export_dict,
RealmAuditLog.objects.filter(
realm=admin.realm,
event_type=RealmAuditLog.REALM_EXPORTED).count(),
)
# Finally, delete the file.
result = self.client_delete(f"/json/export/realm/{audit_log_entry.id}")
self.assert_json_success(result)
response = self.client_get(export_path)
self.assertEqual(response.status_code, 404)
# Try to delete an export with a `deleted_timestamp` key.
audit_log_entry.refresh_from_db()
extra_data = audit_log_entry.extra_data
assert extra_data is not None
export_data = orjson.loads(extra_data)
self.assertIn("deleted_timestamp", export_data)
result = self.client_delete(f"/json/export/realm/{audit_log_entry.id}")
self.assert_json_error(result, "Export already deleted")
# Now try to delete a non-existent export.
result = self.client_delete("/json/export/realm/0")
self.assert_json_error(result, "Invalid data export ID")
