def test_FileDownloadJob_bad_sha256_etag(mocker, homedir, session,
session_maker):
source = factory.Source()
file_ = factory.File(source=source, is_downloaded=None, is_decrypted=None)
session.add(source)
session.add(file_)
session.commit()
gpg = GpgHelper(homedir, session_maker, is_qubes=False)
def fake_download(sdk_obj: SdkSubmission, timeout: int) -> Tuple[str, str]:
'''
:return: (etag, path_to_dl)
'''
full_path = os.path.join(homedir, 'data', 'mock')
with open(full_path, 'wb') as f:
f.write(b'')
return ('sha256:not-a-sha-sum', full_path)
api_client = mocker.MagicMock()
api_client.default_request_timeout = mocker.MagicMock()
api_client.download_submission = fake_download
job = FileDownloadJob(
file_.uuid,
os.path.join(homedir, 'data'),
gpg,
)
with pytest.raises(DownloadChecksumMismatchException):
job.call_api(api_client, session)
def test_FileDownloadJob_happy_path_unknown_etag(mocker, homedir, session,
session_maker):
source = factory.Source()
file_ = factory.File(source=source, is_downloaded=None, is_decrypted=None)
session.add(source)
session.add(file_)
session.commit()
gpg = GpgHelper(homedir, session_maker, is_qubes=False)
def fake_download(sdk_obj: SdkSubmission, timeout: int) -> Tuple[str, str]:
"""
:return: (etag, path_to_dl)
"""
full_path = os.path.join(homedir, "data", "mock")
with open(full_path, "wb") as f:
f.write(b"")
return ("UNKNOWN:abc123", full_path)
api_client = mocker.MagicMock()
api_client.default_request_timeout = mocker.MagicMock()
api_client.download_submission = fake_download
job = FileDownloadJob(file_.uuid, os.path.join(homedir, "data"), gpg)
mock_decrypt = patch_decrypt(mocker, homedir, gpg, file_.filename)
mock_logger = mocker.patch("securedrop_client.api_jobs.downloads.logger")
job.call_api(api_client, session)
log_msg = mock_logger.debug.call_args_list[0][0][0]
assert log_msg.startswith("Unknown hash algorithm")
# ensure mocks aren't stale
assert mock_decrypt.called
def test_ApiJobQueue_enqueue_when_queues_are_running(mocker):
mock_client = mocker.MagicMock()
mock_session_maker = mocker.MagicMock()
job_queue = ApiJobQueue(mock_client, mock_session_maker)
job_priority = 2
dummy_job = factory.dummy_job_factory(mocker, 'mock')()
job_queue.JOB_PRIORITIES = {
FileDownloadJob: job_priority,
type(dummy_job): job_priority
}
mock_download_file_queue = mocker.patch.object(job_queue,
'download_file_queue')
mock_main_queue = mocker.patch.object(job_queue, 'main_queue')
mock_download_file_add_job = mocker.patch.object(mock_download_file_queue,
'add_job')
mock_main_queue_add_job = mocker.patch.object(mock_main_queue, 'add_job')
job_queue.main_queue.api_client = 'has a value'
job_queue.download_file_queue.api_client = 'has a value'
job_queue.main_thread.isRunning = mocker.MagicMock(return_value=True)
job_queue.download_file_thread.isRunning = mocker.MagicMock(
return_value=True)
dl_job = FileDownloadJob('mock', 'mock', 'mock')
job_queue.enqueue(dl_job)
mock_download_file_add_job.assert_called_once_with(dl_job)
assert not mock_main_queue_add_job.called
# reset for next test
mock_download_file_queue.reset_mock()
mock_download_file_add_job.reset_mock()
mock_main_queue.reset_mock()
mock_main_queue_add_job.reset_mock()
job_queue.enqueue(FileDownloadJob('mock', 'mock', 'mock'))
assert not mock_main_queue_add_job.called
# reset for next test
mock_download_file_queue.reset_mock()
mock_download_file_add_job.reset_mock()
mock_main_queue.reset_mock()
mock_main_queue_add_job.reset_mock()
job_queue.enqueue(dummy_job)
mock_main_queue_add_job.assert_called_once_with(dummy_job)
assert not mock_download_file_add_job.called
def test_ApiJobQueue_enqueue_when_queues_are_not_running(mocker):
mock_client = mocker.MagicMock()
mock_session_maker = mocker.MagicMock()
job_queue = ApiJobQueue(mock_client, mock_session_maker)
job_priority = 2
dummy_job = factory.dummy_job_factory(mocker, 'mock')()
job_queue.JOB_PRIORITIES = {
FileDownloadJob: job_priority,
type(dummy_job): job_priority
}
mock_download_file_queue = mocker.patch.object(job_queue,
'download_file_queue')
mock_main_queue = mocker.patch.object(job_queue, 'main_queue')
mock_download_file_add_job = mocker.patch.object(mock_download_file_queue,
'add_job')
mock_main_queue_add_job = mocker.patch.object(mock_main_queue, 'add_job')
job_queue.main_queue.api_client = 'has a value'
job_queue.download_file_queue.api_client = 'has a value'
job_queue.stop(
) # queues are already not running, but just in case the code changes one day
dl_job = FileDownloadJob('mock', 'mock', 'mock')
job_queue.enqueue(dl_job)
mock_download_file_add_job.assert_not_called()
mock_main_queue_add_job.assert_not_called()
def test_RunnableQueue_duplicate_jobs(mocker):
"""
Verify that duplicate jobs are not added to the queue.
"""
mock_api_client = mocker.MagicMock()
mock_session = mocker.MagicMock()
mock_session_maker = mocker.MagicMock(return_value=mock_session)
dl_job = FileDownloadJob("mock", "mock", "mock")
msg_dl_job = MessageDownloadJob("mock", "mock", "mock")
queue = RunnableQueue(mock_api_client, mock_session_maker)
debug_logger = mocker.patch("securedrop_client.queue.logger.debug")
# Queue begins empty (0 entries).
assert len(queue.queue.queue) == 0
queue.add_job(dl_job)
assert len(queue.queue.queue) == 1
# Now add the same job again.
queue.add_job(dl_job)
assert len(queue.queue.queue) == 1
log_msg = "Duplicate job {}, skipping".format(dl_job)
debug_logger.call_args[1] == log_msg
# Now add a _different_ job with the same arguments (same uuid).
queue.add_job(msg_dl_job)
assert len(queue.queue.queue) == 2
# Ensure that using _re_add_job in the case of a timeout won't allow duplicate
# jobs to be added.
with queue.condition_add_or_remove_job:
queue._re_add_job(msg_dl_job)
assert len(queue.queue.queue) == 2
def _submit_download_job(self, object_type: Union[Type[db.Reply],
Type[db.Message],
Type[db.File]],
uuid: str) -> None:
if object_type == db.Reply:
job = ReplyDownloadJob(
uuid, self.data_dir, self.gpg
) # type: Union[ReplyDownloadJob, MessageDownloadJob, FileDownloadJob]
job.success_signal.connect(self.on_reply_download_success,
type=Qt.QueuedConnection)
job.failure_signal.connect(self.on_reply_download_failure,
type=Qt.QueuedConnection)
elif object_type == db.Message:
job = MessageDownloadJob(uuid, self.data_dir, self.gpg)
job.success_signal.connect(self.on_message_download_success,
type=Qt.QueuedConnection)
job.failure_signal.connect(self.on_message_download_failure,
type=Qt.QueuedConnection)
elif object_type == db.File:
job = FileDownloadJob(uuid, self.data_dir, self.gpg)
job.success_signal.connect(self.on_file_download_success,
type=Qt.QueuedConnection)
job.failure_signal.connect(self.on_file_download_failure,
type=Qt.QueuedConnection)
self.api_job_queue.enqueue(job)
def test_FileDownloadJob_decryption_error(mocker, homedir, session,
session_maker):
source = factory.Source()
file_ = factory.File(source=source, is_downloaded=None, is_decrypted=None)
session.add(source)
session.add(file_)
session.commit()
gpg = GpgHelper(homedir, session_maker, is_qubes=False)
mock_decrypt = mocker.patch.object(gpg,
'decrypt_submission_or_reply',
side_effect=CryptoError)
def fake_download(sdk_obj: SdkSubmission, timeout: int) -> Tuple[str, str]:
'''
:return: (etag, path_to_dl)
'''
full_path = os.path.join(homedir, 'data', 'mock')
with open(full_path, 'wb') as f:
f.write(b'wat')
# sha256 of b'wat'
return (
'sha256:f00a787f7492a95e165b470702f4fe9373583fbdc025b2c8bdf0262cc48fcff4',
full_path)
api_client = mocker.MagicMock()
api_client.default_request_timeout = mocker.MagicMock()
api_client.download_submission = fake_download
job = FileDownloadJob(
file_.uuid,
os.path.join(homedir, 'data'),
gpg,
)
mock_logger = mocker.patch('securedrop_client.api_jobs.downloads.logger')
with pytest.raises(DownloadDecryptionException):
job.call_api(api_client, session)
log_msg = mock_logger.debug.call_args_list[0][0][0]
assert log_msg.startswith('Failed to decrypt file')
# ensure mocks aren't stale
assert mock_decrypt.called
def test_FileDownloadJob_message_already_downloaded(mocker, homedir, session,
session_maker):
"""
Test that call_api just decrypts and returns uuid if already downloaded.
"""
file_ = factory.File(source=factory.Source(),
is_downloaded=True,
is_decrypted=False)
session.add(file_)
session.commit()
gpg = GpgHelper(homedir, session_maker, is_qubes=False)
job = FileDownloadJob(file_.uuid, os.path.join(homedir, 'data'), gpg)
patch_decrypt(mocker, homedir, gpg, file_.filename)
api_client = mocker.MagicMock()
download_fn = mocker.patch.object(api_client, 'download_submission')
return_uuid = job.call_api(api_client, session)
assert file_.uuid == return_uuid
assert file_.is_decrypted is True
download_fn.assert_not_called()
def test_FileDownloadJob_message_already_decrypted(mocker, homedir, session,
session_maker):
"""
Test that call_api just returns uuid if already decrypted.
"""
file_ = factory.File(source=factory.Source(),
is_downloaded=True,
is_decrypted=True)
session.add(file_)
session.commit()
gpg = GpgHelper(homedir, session_maker, is_qubes=False)
job = FileDownloadJob(file_.uuid, homedir, gpg)
decrypt_fn = mocker.patch.object(job.gpg, 'decrypt_submission_or_reply')
api_client = mocker.MagicMock()
download_fn = mocker.patch.object(api_client, 'download_submission')
return_uuid = job.call_api(api_client, session)
assert file_.uuid == return_uuid
decrypt_fn.assert_not_called()
download_fn.assert_not_called()
def test_FileDownloadJob_happy_path_no_etag(mocker, homedir, session,
session_maker):
source = factory.Source()
file_ = factory.File(source=source, is_downloaded=False, is_decrypted=None)
session.add(source)
session.add(file_)
session.commit()
gpg = GpgHelper(homedir, session_maker, is_qubes=False)
mock_decrypt = patch_decrypt(mocker, homedir, gpg, file_.filename)
def fake_download(sdk_obj: SdkSubmission, timeout: int) -> Tuple[str, str]:
'''
:return: (etag, path_to_dl)
'''
full_path = os.path.join(homedir, 'data', 'mock')
with open(full_path, 'wb') as f:
f.write(b'')
return ('', full_path)
api_client = mocker.MagicMock()
api_client.default_request_timeout = mocker.MagicMock()
api_client.download_submission = fake_download
job = FileDownloadJob(
file_.uuid,
os.path.join(homedir, 'data'),
gpg,
)
mock_logger = mocker.patch('securedrop_client.api_jobs.downloads.logger')
job.call_api(api_client, session)
log_msg = mock_logger.debug.call_args_list[0][0][0]
assert log_msg.startswith('No ETag. Skipping integrity check')
# ensure mocks aren't stale
assert mock_decrypt.called
def test_FileDownloadJob_happy_path_sha256_etag(mocker, homedir, session,
session_maker):
source = factory.Source()
file_ = factory.File(source=source, is_downloaded=None, is_decrypted=None)
session.add(source)
session.add(file_)
session.commit()
gpg = GpgHelper(homedir, session_maker, is_qubes=False)
mock_decrypt = patch_decrypt(mocker, homedir, gpg, file_.filename)
def fake_download(sdk_obj: SdkSubmission, timeout: int) -> Tuple[str, str]:
'''
:return: (etag, path_to_dl)
'''
full_path = os.path.join(homedir, 'data', 'mock')
with open(full_path, 'wb') as f:
f.write(b'wat')
# sha256 of b'wat'
return (
'sha256:f00a787f7492a95e165b470702f4fe9373583fbdc025b2c8bdf0262cc48fcff4',
full_path)
api_client = mocker.MagicMock()
api_client.default_request_timeout = mocker.MagicMock()
api_client.download_submission = fake_download
job = FileDownloadJob(
file_.uuid,
os.path.join(homedir, 'data'),
gpg,
)
job.call_api(api_client, session)
# ensure mocks aren't stale
assert mock_decrypt.called
def test_FileDownloadJob_decryption_error(mocker, homedir, session,
session_maker, download_error_codes):
source = factory.Source()
file_ = factory.File(source=source, is_downloaded=None, is_decrypted=None)
session.add(source)
session.add(file_)
session.commit()
gpg = GpgHelper(homedir, session_maker, is_qubes=False)
mock_decrypt = mocker.patch.object(gpg,
"decrypt_submission_or_reply",
side_effect=CryptoError)
def fake_download(sdk_obj: SdkSubmission, timeout: int) -> Tuple[str, str]:
"""
:return: (etag, path_to_dl)
"""
full_path = os.path.join(homedir, "data", "mock")
with open(full_path, "wb") as f:
f.write(b"wat")
# sha256 of b'wat'
return (
"sha256:f00a787f7492a95e165b470702f4fe9373583fbdc025b2c8bdf0262cc48fcff4",
full_path,
)
api_client = mocker.MagicMock()
api_client.default_request_timeout = mocker.MagicMock()
api_client.download_submission = fake_download
job = FileDownloadJob(file_.uuid, os.path.join(homedir, "data"), gpg)
with pytest.raises(DownloadDecryptionException):
job.call_api(api_client, session)
# ensure mocks aren't stale
assert mock_decrypt.called
def test_timeout_length_of_file_downloads(mocker, homedir, session,
session_maker):
"""
Ensure that files downloads have timeouts scaled by the size of the file.
"""
source = factory.Source()
small_file = factory.File(source=source,
is_downloaded=None,
is_decrypted=None,
size=1)
large_file = factory.File(source=source,
is_downloaded=None,
is_decrypted=None,
size=100)
session.add(source)
session.add(small_file)
session.add(large_file)
session.commit()
gpg = GpgHelper(homedir, session_maker, is_qubes=False)
small_job = FileDownloadJob(
small_file.uuid,
os.path.join(homedir, 'data'),
gpg,
)
large_job = FileDownloadJob(
small_file.uuid,
os.path.join(homedir, 'data'),
gpg,
)
small_file_timeout = small_job._get_realistic_timeout(small_file.size)
large_file_timeout = large_job._get_realistic_timeout(large_file.size)
assert small_file_timeout < large_file_timeout
def test_timeout_length_of_file_downloads(mocker, homedir, session,
session_maker):
"""
Ensure that files downloads have timeouts scaled by the size of the file.
"""
source = factory.Source()
zero_byte_file = factory.File(source=source,
is_downloaded=None,
is_decrypted=None,
size=0)
one_byte_file = factory.File(source=source,
is_downloaded=None,
is_decrypted=None,
size=1)
KB_file = factory.File(source=source,
is_downloaded=None,
is_decrypted=None,
size=1000)
fifty_KB_file = factory.File(source=source,
is_downloaded=None,
is_decrypted=None,
size=50000)
half_MB_file = factory.File(source=source,
is_downloaded=None,
is_decrypted=None,
size=500000)
MB_file = factory.File(source=source,
is_downloaded=None,
is_decrypted=None,
size=1000000)
five_MB_file = factory.File(source=source,
is_downloaded=None,
is_decrypted=None,
size=5000000)
haf_GB_file = factory.File(source=source,
is_downloaded=None,
is_decrypted=None,
size=500000000)
GB_file = factory.File(source=source,
is_downloaded=None,
is_decrypted=None,
size=1000000000)
session.add(source)
session.add(zero_byte_file)
session.add(one_byte_file)
session.add(KB_file)
session.add(fifty_KB_file)
session.add(half_MB_file)
session.add(MB_file)
session.add(five_MB_file)
session.add(haf_GB_file)
session.add(GB_file)
session.commit()
gpg = GpgHelper(homedir, session_maker, is_qubes=False)
zero_byte_file_job = FileDownloadJob(zero_byte_file.uuid,
os.path.join(homedir, 'data'), gpg)
one_byte_file_job = FileDownloadJob(one_byte_file.uuid,
os.path.join(homedir, 'data'), gpg)
KB_file_job = FileDownloadJob(KB_file.uuid, os.path.join(homedir, 'data'),
gpg)
fifty_KB_file_job = FileDownloadJob(fifty_KB_file.uuid,
os.path.join(homedir, 'data'), gpg)
half_MB_file_job = FileDownloadJob(half_MB_file.uuid,
os.path.join(homedir, 'data'), gpg)
MB_file_job = FileDownloadJob(MB_file.uuid, os.path.join(homedir, 'data'),
gpg)
five_MB_file_job = FileDownloadJob(five_MB_file.uuid,
os.path.join(homedir, 'data'), gpg)
half_GB_file_job = FileDownloadJob(haf_GB_file.uuid,
os.path.join(homedir, 'data'), gpg)
GB_file_job = FileDownloadJob(GB_file.uuid, os.path.join(homedir, 'data'),
gpg)
zero_byte_file_timeout = zero_byte_file_job._get_realistic_timeout(
zero_byte_file.size)
one_byte_file_timeout = one_byte_file_job._get_realistic_timeout(
one_byte_file.size)
KB_file_timeout = KB_file_job._get_realistic_timeout(KB_file.size)
fifty_KB_file_timeout = fifty_KB_file_job._get_realistic_timeout(
fifty_KB_file.size)
half_MB_file_timeout = half_MB_file_job._get_realistic_timeout(
half_MB_file.size)
MB_file_timeout = MB_file_job._get_realistic_timeout(MB_file.size)
five_MB_file_timeout = five_MB_file_job._get_realistic_timeout(
five_MB_file.size)
GB_file_timeout = GB_file_job._get_realistic_timeout(GB_file.size)
half_GB_file_timeout = half_GB_file_job._get_realistic_timeout(
haf_GB_file.size)
# timeout = ceil((file_size/100000)*1.5)+25
assert zero_byte_file_timeout == 25
assert one_byte_file_timeout == 26
assert KB_file_timeout == 26
assert fifty_KB_file_timeout == 26
assert half_MB_file_timeout == 33
assert MB_file_timeout == 40
assert five_MB_file_timeout == 100
assert half_GB_file_timeout == 7525
assert GB_file_timeout == 15025