def test_source_collection_ordering_with_multiple_draft_replies():
# Create some test submissions, replies, and draft replies.
source = factory.Source()
file_1 = File(source=source, uuid="test", size=123, filename="1-test.doc.gpg",
download_url='http://test/test')
message_2 = Message(source=source, uuid="test", size=123, filename="2-test.doc.gpg",
download_url='http://test/test')
user = User(username='******')
reply_3 = Reply(source=source, journalist=user, filename="3-reply.gpg",
size=1234, uuid='test')
draft_reply_4 = DraftReply(uuid='4', source=source, journalist=user, file_counter=3,
timestamp=datetime.datetime(2000, 6, 6, 6, 0))
draft_reply_5 = DraftReply(uuid='5', source=source, journalist=user, file_counter=3,
timestamp=datetime.datetime(2001, 6, 6, 6, 0))
reply_6 = Reply(source=source, journalist=user, filename="4-reply.gpg",
size=1234, uuid='test2')
draft_reply_7 = DraftReply(uuid='6', source=source, journalist=user, file_counter=4,
timestamp=datetime.datetime(2002, 6, 6, 6, 0))
source.files = [file_1]
source.messages = [message_2]
source.replies = [reply_3, reply_6]
source.draftreplies = [draft_reply_4, draft_reply_5, draft_reply_7]
# Now these items should be in the source collection in the proper order
assert source.collection[0] == file_1
assert source.collection[1] == message_2
assert source.collection[2] == reply_3
assert source.collection[3] == draft_reply_4
assert source.collection[4] == draft_reply_5
assert source.collection[5] == reply_6
assert source.collection[6] == draft_reply_7
def test_string_representation_of_reply():
user = User('hehe')
source = factory.Source()
reply = Reply(source=source,
journalist=user,
filename="reply.gpg",
size=1234,
uuid='test')
reply.__repr__()
def test_repr_representation_of_reply():
user = User(username="******")
source = factory.Source()
reply = Reply(source=source,
journalist=user,
filename="1-reply.gpg",
size=1234,
uuid="test")
reply.__repr__()
def test_reply_init():
'''
Check that:
- We can't pass the file_counter attribute
- The file_counter attribute is see correctly based off the filename
'''
with pytest.raises(TypeError):
Reply(file_counter=1)
r = Reply(filename="1-foo")
assert r.file_counter == 1
def call_api(self, api_client: API, session: Session) -> str:
'''
Override ApiJob.
Encrypt the reply and send it to the server. If the call is successful, add it to the local
database and return the reply uuid string. Otherwise raise a SendReplyJobException so that
we can return the reply uuid.
'''
try:
encrypted_reply = self.gpg.encrypt_to_source(self.source_uuid, self.message)
sdk_reply = self._make_call(encrypted_reply, api_client)
source = session.query(Source).filter_by(uuid=self.source_uuid).one()
reply_db_object = Reply(
uuid=self.reply_uuid,
source_id=source.id,
journalist_id=api_client.token_journalist_uuid,
filename=sdk_reply.filename,
content=self.message,
is_downloaded=True,
is_decrypted=True
)
session.add(reply_db_object)
session.commit()
return reply_db_object.uuid
except RequestTimeoutError as e:
message = "Failed to send reply for source {id} due to Exception: {error}".format(
id=self.source_uuid, error=e)
raise SendReplyJobTimeoutError(message, self.reply_uuid)
except Exception as e:
message = "Failed to send reply for source {id} due to Exception: {error}".format(
id=self.source_uuid, error=e)
raise SendReplyJobError(message, self.reply_uuid)
def test_source_server_collection():
# Create some test submissions and replies
source = factory.Source()
file_ = File(source=source, uuid="test", size=123, filename="2-test.doc.gpg",
download_url='http://test/test')
message = Message(source=source, uuid="test", size=123, filename="3-test.doc.gpg",
download_url='http://test/test')
user = User(username='******')
reply = Reply(source=source, journalist=user, filename="1-reply.gpg",
size=1234, uuid='test')
draft_reply = DraftReply(source=source, journalist=user,
uuid='test',
timestamp=datetime.datetime(2002, 6, 6, 6, 0))
source.files = [file_]
source.messages = [message]
source.replies = [reply]
source.draftreplies = [draft_reply]
# Now these items should be in the source collection in the proper order
assert source.server_collection[0] == reply
assert source.server_collection[1] == file_
assert source.server_collection[2] == message
# Drafts do not appear in the server_collection, they are local only.
assert draft_reply not in source.server_collection
def test_source_collection():
# Create some test submissions and replies
source = factory.Source()
file_ = File(
source=source,
uuid="test",
size=123,
filename="2-test.doc.gpg",
download_url="http://test/test",
)
message = Message(
source=source,
uuid="test",
size=123,
filename="3-test.doc.gpg",
download_url="http://test/test",
)
user = User(username="******")
reply = Reply(source=source,
journalist=user,
filename="1-reply.gpg",
size=1234,
uuid="test")
source.files = [file_]
source.messages = [message]
source.replies = [reply]
# Now these items should be in the source collection in the proper order
assert source.collection[0] == reply
assert source.collection[1] == file_
assert source.collection[2] == message
def update_replies(remote_replies: List[SDKReply], local_replies: List[Reply],
session: Session, data_dir: str) -> None:
"""
* Existing replies are updated in the local database.
* New replies have an entry created in the local database.
* Local replies not returned in the remote replies are deleted from the
local database.
If a reply references a new journalist username, add them to the database
as a new user.
"""
local_uuids = {reply.uuid for reply in local_replies}
for reply in remote_replies:
if reply.uuid in local_uuids:
local_reply = [r for r in local_replies if r.uuid == reply.uuid][0]
# Update files on disk to match new filename.
if (local_reply.filename != reply.filename):
rename_file(data_dir, local_reply.filename, reply.filename)
# Update an existing record.
user = find_or_create_user(reply.journalist_uuid,
reply.journalist_username, session)
local_reply.journalist_id = user.id
local_reply.filename = reply.filename
local_reply.size = reply.size
local_uuids.remove(reply.uuid)
logger.debug('Updated reply {}'.format(reply.uuid))
else:
# A new reply to be added to the database.
source_uuid = reply.source_uuid
source = session.query(Source).filter_by(uuid=source_uuid)[0]
user = find_or_create_user(reply.journalist_uuid,
reply.journalist_username, session)
nr = Reply(uuid=reply.uuid,
journalist_id=user.id,
source_id=source.id,
filename=reply.filename,
size=reply.size)
session.add(nr)
logger.debug('Added new reply {}'.format(reply.uuid))
# The uuids remaining in local_uuids do not exist on the remote server, so
# delete the related records.
for deleted_reply in [r for r in local_replies if r.uuid in local_uuids]:
delete_single_submission_or_reply_on_disk(deleted_reply, data_dir)
session.delete(deleted_reply)
logger.debug('Deleted reply {}'.format(deleted_reply.uuid))
session.commit()
def test_source_collection():
# Create some test submissions and replies
source = factory.Source()
submission = Submission(source=source,
uuid="test",
size=123,
filename="2-test.doc.gpg",
download_url='http://test/test')
user = User('hehe')
reply = Reply(source=source,
journalist=user,
filename="1-reply.gpg",
size=1234,
uuid='test')
source.submissions = [submission]
source.replies = [reply]
# Now these items should be in the source collection in the proper order
assert source.collection[0] == reply
assert source.collection[1] == submission
def update_replies(remote_replies: List[SDKReply], local_replies: List[Reply],
session: Session, data_dir: str) -> None:
"""
* Existing replies are updated in the local database.
* New replies have an entry created in the local database.
* Local replies not returned in the remote replies are deleted from the
local database unless they are pending or failed.
If a reply references a new journalist username, add them to the database
as a new user.
"""
local_uuids = {reply.uuid for reply in local_replies}
for reply in remote_replies:
if reply.uuid in local_uuids:
local_reply = [r for r in local_replies if r.uuid == reply.uuid][0]
user = find_or_create_user(reply.journalist_uuid, reply.journalist_username, session)
local_reply.journalist_id = user.id
local_reply.size = reply.size
local_uuids.remove(reply.uuid)
logger.debug('Updated reply {}'.format(reply.uuid))
else:
# A new reply to be added to the database.
source_uuid = reply.source_uuid
source = session.query(Source).filter_by(uuid=source_uuid)[0]
user = find_or_create_user(
reply.journalist_uuid,
reply.journalist_username,
session)
nr = Reply(uuid=reply.uuid,
journalist_id=user.id,
source_id=source.id,
filename=reply.filename,
size=reply.size)
session.add(nr)
# All replies fetched from the server have succeeded in being sent,
# so we should delete the corresponding draft locally if it exists.
try:
draft_reply_db_object = session.query(DraftReply).filter_by(
uuid=reply.uuid).one()
update_draft_replies(session, draft_reply_db_object.source.id,
draft_reply_db_object.timestamp,
draft_reply_db_object.file_counter,
nr.file_counter)
session.delete(draft_reply_db_object)
except NoResultFound:
pass # No draft locally stored corresponding to this reply.
logger.debug('Added new reply {}'.format(reply.uuid))
# The uuids remaining in local_uuids do not exist on the remote server, so
# delete the related records.
replies_to_delete = [r for r in local_replies if r.uuid in local_uuids]
for deleted_reply in replies_to_delete:
delete_single_submission_or_reply_on_disk(deleted_reply, data_dir)
session.delete(deleted_reply)
logger.debug('Deleted reply {}'.format(deleted_reply.uuid))
session.commit()
def call_api(self, api_client: API, session: Session) -> str:
'''
Override ApiJob.
Encrypt the reply and send it to the server. If the call is successful, add it to the local
database and return the reply uuid string. Otherwise raise a SendReplyJobException so that
we can return the reply uuid.
'''
try:
# If the reply has already made it to the server but we didn't get a 201 response back,
# then a reply with self.reply_uuid will exist in the replies table.
reply_db_object = session.query(Reply).filter_by(
uuid=self.reply_uuid).one_or_none()
if reply_db_object:
logger.debug(
'Reply {} has already been sent successfully'.format(
self.reply_uuid))
return reply_db_object.uuid
# If the draft does not exist because it was deleted locally then do not send the
# message to the source.
draft_reply_db_object = session.query(DraftReply).filter_by(
uuid=self.reply_uuid).one_or_none()
if not draft_reply_db_object:
raise Exception('Draft reply {} does not exist'.format(
self.reply_uuid))
# If the source was deleted locally then do not send the message and delete the draft.
source = session.query(Source).filter_by(
uuid=self.source_uuid).one_or_none()
if not source:
session.delete(draft_reply_db_object)
session.commit()
raise Exception('Source {} does not exists'.format(
self.source_uuid))
# Send the draft reply to the source
encrypted_reply = self.gpg.encrypt_to_source(
self.source_uuid, self.message)
sdk_reply = self._make_call(encrypted_reply, api_client)
# Create a new reply object with an updated filename and file counter
interaction_count = source.interaction_count + 1
filename = '{}-{}-reply.gpg'.format(interaction_count,
source.journalist_designation)
reply_db_object = Reply(
uuid=self.reply_uuid,
source_id=source.id,
filename=filename,
journalist_id=api_client.token_journalist_uuid,
content=self.message,
is_downloaded=True,
is_decrypted=True)
new_file_counter = int(sdk_reply.filename.split('-')[0])
reply_db_object.file_counter = new_file_counter
reply_db_object.filename = sdk_reply.filename
# Update following draft replies for the same source to reflect the new reply count
draft_file_counter = draft_reply_db_object.file_counter
draft_timestamp = draft_reply_db_object.timestamp
update_draft_replies(session,
source.id,
draft_timestamp,
draft_file_counter,
new_file_counter,
commit=False)
# Add reply to replies table and increase the source interaction count by 1 and delete
# the draft reply.
session.add(reply_db_object)
source.interaction_count += 1
session.add(source)
session.delete(draft_reply_db_object)
session.commit()
return reply_db_object.uuid
except (RequestTimeoutError, ServerConnectionError) as e:
message = "Failed to send reply for source {id} due to Exception: {error}".format(
id=self.source_uuid, error=e)
raise SendReplyJobTimeoutError(message, self.reply_uuid)
except Exception as e:
# Continue to store the draft reply
message = '''
Failed to send reply {uuid} for source {id} due to Exception: {error}
'''.format(uuid=self.reply_uuid, id=self.source_uuid, error=e)
self._set_status_to_failed(session)
raise SendReplyJobError(message, self.reply_uuid)
def update_replies(remote_replies: List[SDKReply], local_replies: List[Reply],
session: Session, data_dir: str) -> None:
"""
* Existing replies are updated in the local database.
* New replies have an entry created in the local database.
* Local replies not returned in the remote replies are deleted from the
local database unless they are pending or failed.
If a reply references a new journalist username, add them to the database
as a new user.
"""
local_replies_by_uuid = {r.uuid: r for r in local_replies}
users: Dict[str, User] = {}
source_cache = SourceCache(session)
for reply in remote_replies:
user = users.get(reply.journalist_uuid)
if not user:
user = find_or_create_user(reply.journalist_uuid,
reply.journalist_username, session)
users[reply.journalist_uuid] = user
local_reply = local_replies_by_uuid.get(reply.uuid)
if local_reply:
lazy_setattr(local_reply, "journalist_id", user.id)
lazy_setattr(local_reply, "size", reply.size)
lazy_setattr(local_reply, "filename", reply.filename)
del local_replies_by_uuid[reply.uuid]
logger.debug('Updated reply {}'.format(reply.uuid))
else:
# A new reply to be added to the database.
source = source_cache.get(reply.source_uuid)
if not source:
logger.error(f"No source found for reply {reply.uuid}")
continue
nr = Reply(uuid=reply.uuid,
journalist_id=user.id,
source_id=source.id,
filename=reply.filename,
size=reply.size)
session.add(nr)
# All replies fetched from the server have succeeded in being sent,
# so we should delete the corresponding draft locally if it exists.
try:
draft_reply_db_object = session.query(DraftReply).filter_by(
uuid=reply.uuid).one()
update_draft_replies(session,
draft_reply_db_object.source.id,
draft_reply_db_object.timestamp,
draft_reply_db_object.file_counter,
nr.file_counter,
commit=False)
session.delete(draft_reply_db_object)
except NoResultFound:
pass # No draft locally stored corresponding to this reply.
logger.debug('Added new reply {}'.format(reply.uuid))
# The uuids remaining in local_uuids do not exist on the remote server, so
# delete the related records.
for deleted_reply in local_replies_by_uuid.values():
delete_single_submission_or_reply_on_disk(deleted_reply, data_dir)
session.delete(deleted_reply)
logger.debug('Deleted reply {}'.format(deleted_reply.uuid))
session.commit()