def test_source_collection_ordering_with_multiple_draft_replies(): # Create some test submissions, replies, and draft replies. source = factory.Source() file_1 = File(source=source, uuid="test", size=123, filename="1-test.doc.gpg", download_url='http://test/test') message_2 = Message(source=source, uuid="test", size=123, filename="2-test.doc.gpg", download_url='http://test/test') user = User(username='******') reply_3 = Reply(source=source, journalist=user, filename="3-reply.gpg", size=1234, uuid='test') draft_reply_4 = DraftReply(uuid='4', source=source, journalist=user, file_counter=3, timestamp=datetime.datetime(2000, 6, 6, 6, 0)) draft_reply_5 = DraftReply(uuid='5', source=source, journalist=user, file_counter=3, timestamp=datetime.datetime(2001, 6, 6, 6, 0)) reply_6 = Reply(source=source, journalist=user, filename="4-reply.gpg", size=1234, uuid='test2') draft_reply_7 = DraftReply(uuid='6', source=source, journalist=user, file_counter=4, timestamp=datetime.datetime(2002, 6, 6, 6, 0)) source.files = [file_1] source.messages = [message_2] source.replies = [reply_3, reply_6] source.draftreplies = [draft_reply_4, draft_reply_5, draft_reply_7] # Now these items should be in the source collection in the proper order assert source.collection[0] == file_1 assert source.collection[1] == message_2 assert source.collection[2] == reply_3 assert source.collection[3] == draft_reply_4 assert source.collection[4] == draft_reply_5 assert source.collection[5] == reply_6 assert source.collection[6] == draft_reply_7
def test_string_representation_of_reply(): user = User('hehe') source = factory.Source() reply = Reply(source=source, journalist=user, filename="reply.gpg", size=1234, uuid='test') reply.__repr__()
def test_repr_representation_of_reply(): user = User(username="******") source = factory.Source() reply = Reply(source=source, journalist=user, filename="1-reply.gpg", size=1234, uuid="test") reply.__repr__()
def test_reply_init(): ''' Check that: - We can't pass the file_counter attribute - The file_counter attribute is see correctly based off the filename ''' with pytest.raises(TypeError): Reply(file_counter=1) r = Reply(filename="1-foo") assert r.file_counter == 1
def call_api(self, api_client: API, session: Session) -> str: ''' Override ApiJob. Encrypt the reply and send it to the server. If the call is successful, add it to the local database and return the reply uuid string. Otherwise raise a SendReplyJobException so that we can return the reply uuid. ''' try: encrypted_reply = self.gpg.encrypt_to_source(self.source_uuid, self.message) sdk_reply = self._make_call(encrypted_reply, api_client) source = session.query(Source).filter_by(uuid=self.source_uuid).one() reply_db_object = Reply( uuid=self.reply_uuid, source_id=source.id, journalist_id=api_client.token_journalist_uuid, filename=sdk_reply.filename, content=self.message, is_downloaded=True, is_decrypted=True ) session.add(reply_db_object) session.commit() return reply_db_object.uuid except RequestTimeoutError as e: message = "Failed to send reply for source {id} due to Exception: {error}".format( id=self.source_uuid, error=e) raise SendReplyJobTimeoutError(message, self.reply_uuid) except Exception as e: message = "Failed to send reply for source {id} due to Exception: {error}".format( id=self.source_uuid, error=e) raise SendReplyJobError(message, self.reply_uuid)
def test_source_server_collection(): # Create some test submissions and replies source = factory.Source() file_ = File(source=source, uuid="test", size=123, filename="2-test.doc.gpg", download_url='http://test/test') message = Message(source=source, uuid="test", size=123, filename="3-test.doc.gpg", download_url='http://test/test') user = User(username='******') reply = Reply(source=source, journalist=user, filename="1-reply.gpg", size=1234, uuid='test') draft_reply = DraftReply(source=source, journalist=user, uuid='test', timestamp=datetime.datetime(2002, 6, 6, 6, 0)) source.files = [file_] source.messages = [message] source.replies = [reply] source.draftreplies = [draft_reply] # Now these items should be in the source collection in the proper order assert source.server_collection[0] == reply assert source.server_collection[1] == file_ assert source.server_collection[2] == message # Drafts do not appear in the server_collection, they are local only. assert draft_reply not in source.server_collection
def test_source_collection(): # Create some test submissions and replies source = factory.Source() file_ = File( source=source, uuid="test", size=123, filename="2-test.doc.gpg", download_url="http://test/test", ) message = Message( source=source, uuid="test", size=123, filename="3-test.doc.gpg", download_url="http://test/test", ) user = User(username="******") reply = Reply(source=source, journalist=user, filename="1-reply.gpg", size=1234, uuid="test") source.files = [file_] source.messages = [message] source.replies = [reply] # Now these items should be in the source collection in the proper order assert source.collection[0] == reply assert source.collection[1] == file_ assert source.collection[2] == message
def update_replies(remote_replies: List[SDKReply], local_replies: List[Reply], session: Session, data_dir: str) -> None: """ * Existing replies are updated in the local database. * New replies have an entry created in the local database. * Local replies not returned in the remote replies are deleted from the local database. If a reply references a new journalist username, add them to the database as a new user. """ local_uuids = {reply.uuid for reply in local_replies} for reply in remote_replies: if reply.uuid in local_uuids: local_reply = [r for r in local_replies if r.uuid == reply.uuid][0] # Update files on disk to match new filename. if (local_reply.filename != reply.filename): rename_file(data_dir, local_reply.filename, reply.filename) # Update an existing record. user = find_or_create_user(reply.journalist_uuid, reply.journalist_username, session) local_reply.journalist_id = user.id local_reply.filename = reply.filename local_reply.size = reply.size local_uuids.remove(reply.uuid) logger.debug('Updated reply {}'.format(reply.uuid)) else: # A new reply to be added to the database. source_uuid = reply.source_uuid source = session.query(Source).filter_by(uuid=source_uuid)[0] user = find_or_create_user(reply.journalist_uuid, reply.journalist_username, session) nr = Reply(uuid=reply.uuid, journalist_id=user.id, source_id=source.id, filename=reply.filename, size=reply.size) session.add(nr) logger.debug('Added new reply {}'.format(reply.uuid)) # The uuids remaining in local_uuids do not exist on the remote server, so # delete the related records. for deleted_reply in [r for r in local_replies if r.uuid in local_uuids]: delete_single_submission_or_reply_on_disk(deleted_reply, data_dir) session.delete(deleted_reply) logger.debug('Deleted reply {}'.format(deleted_reply.uuid)) session.commit()
def test_source_collection(): # Create some test submissions and replies source = factory.Source() submission = Submission(source=source, uuid="test", size=123, filename="2-test.doc.gpg", download_url='http://test/test') user = User('hehe') reply = Reply(source=source, journalist=user, filename="1-reply.gpg", size=1234, uuid='test') source.submissions = [submission] source.replies = [reply] # Now these items should be in the source collection in the proper order assert source.collection[0] == reply assert source.collection[1] == submission
def update_replies(remote_replies: List[SDKReply], local_replies: List[Reply], session: Session, data_dir: str) -> None: """ * Existing replies are updated in the local database. * New replies have an entry created in the local database. * Local replies not returned in the remote replies are deleted from the local database unless they are pending or failed. If a reply references a new journalist username, add them to the database as a new user. """ local_uuids = {reply.uuid for reply in local_replies} for reply in remote_replies: if reply.uuid in local_uuids: local_reply = [r for r in local_replies if r.uuid == reply.uuid][0] user = find_or_create_user(reply.journalist_uuid, reply.journalist_username, session) local_reply.journalist_id = user.id local_reply.size = reply.size local_uuids.remove(reply.uuid) logger.debug('Updated reply {}'.format(reply.uuid)) else: # A new reply to be added to the database. source_uuid = reply.source_uuid source = session.query(Source).filter_by(uuid=source_uuid)[0] user = find_or_create_user( reply.journalist_uuid, reply.journalist_username, session) nr = Reply(uuid=reply.uuid, journalist_id=user.id, source_id=source.id, filename=reply.filename, size=reply.size) session.add(nr) # All replies fetched from the server have succeeded in being sent, # so we should delete the corresponding draft locally if it exists. try: draft_reply_db_object = session.query(DraftReply).filter_by( uuid=reply.uuid).one() update_draft_replies(session, draft_reply_db_object.source.id, draft_reply_db_object.timestamp, draft_reply_db_object.file_counter, nr.file_counter) session.delete(draft_reply_db_object) except NoResultFound: pass # No draft locally stored corresponding to this reply. logger.debug('Added new reply {}'.format(reply.uuid)) # The uuids remaining in local_uuids do not exist on the remote server, so # delete the related records. replies_to_delete = [r for r in local_replies if r.uuid in local_uuids] for deleted_reply in replies_to_delete: delete_single_submission_or_reply_on_disk(deleted_reply, data_dir) session.delete(deleted_reply) logger.debug('Deleted reply {}'.format(deleted_reply.uuid)) session.commit()
def call_api(self, api_client: API, session: Session) -> str: ''' Override ApiJob. Encrypt the reply and send it to the server. If the call is successful, add it to the local database and return the reply uuid string. Otherwise raise a SendReplyJobException so that we can return the reply uuid. ''' try: # If the reply has already made it to the server but we didn't get a 201 response back, # then a reply with self.reply_uuid will exist in the replies table. reply_db_object = session.query(Reply).filter_by( uuid=self.reply_uuid).one_or_none() if reply_db_object: logger.debug( 'Reply {} has already been sent successfully'.format( self.reply_uuid)) return reply_db_object.uuid # If the draft does not exist because it was deleted locally then do not send the # message to the source. draft_reply_db_object = session.query(DraftReply).filter_by( uuid=self.reply_uuid).one_or_none() if not draft_reply_db_object: raise Exception('Draft reply {} does not exist'.format( self.reply_uuid)) # If the source was deleted locally then do not send the message and delete the draft. source = session.query(Source).filter_by( uuid=self.source_uuid).one_or_none() if not source: session.delete(draft_reply_db_object) session.commit() raise Exception('Source {} does not exists'.format( self.source_uuid)) # Send the draft reply to the source encrypted_reply = self.gpg.encrypt_to_source( self.source_uuid, self.message) sdk_reply = self._make_call(encrypted_reply, api_client) # Create a new reply object with an updated filename and file counter interaction_count = source.interaction_count + 1 filename = '{}-{}-reply.gpg'.format(interaction_count, source.journalist_designation) reply_db_object = Reply( uuid=self.reply_uuid, source_id=source.id, filename=filename, journalist_id=api_client.token_journalist_uuid, content=self.message, is_downloaded=True, is_decrypted=True) new_file_counter = int(sdk_reply.filename.split('-')[0]) reply_db_object.file_counter = new_file_counter reply_db_object.filename = sdk_reply.filename # Update following draft replies for the same source to reflect the new reply count draft_file_counter = draft_reply_db_object.file_counter draft_timestamp = draft_reply_db_object.timestamp update_draft_replies(session, source.id, draft_timestamp, draft_file_counter, new_file_counter, commit=False) # Add reply to replies table and increase the source interaction count by 1 and delete # the draft reply. session.add(reply_db_object) source.interaction_count += 1 session.add(source) session.delete(draft_reply_db_object) session.commit() return reply_db_object.uuid except (RequestTimeoutError, ServerConnectionError) as e: message = "Failed to send reply for source {id} due to Exception: {error}".format( id=self.source_uuid, error=e) raise SendReplyJobTimeoutError(message, self.reply_uuid) except Exception as e: # Continue to store the draft reply message = ''' Failed to send reply {uuid} for source {id} due to Exception: {error} '''.format(uuid=self.reply_uuid, id=self.source_uuid, error=e) self._set_status_to_failed(session) raise SendReplyJobError(message, self.reply_uuid)
def update_replies(remote_replies: List[SDKReply], local_replies: List[Reply], session: Session, data_dir: str) -> None: """ * Existing replies are updated in the local database. * New replies have an entry created in the local database. * Local replies not returned in the remote replies are deleted from the local database unless they are pending or failed. If a reply references a new journalist username, add them to the database as a new user. """ local_replies_by_uuid = {r.uuid: r for r in local_replies} users: Dict[str, User] = {} source_cache = SourceCache(session) for reply in remote_replies: user = users.get(reply.journalist_uuid) if not user: user = find_or_create_user(reply.journalist_uuid, reply.journalist_username, session) users[reply.journalist_uuid] = user local_reply = local_replies_by_uuid.get(reply.uuid) if local_reply: lazy_setattr(local_reply, "journalist_id", user.id) lazy_setattr(local_reply, "size", reply.size) lazy_setattr(local_reply, "filename", reply.filename) del local_replies_by_uuid[reply.uuid] logger.debug('Updated reply {}'.format(reply.uuid)) else: # A new reply to be added to the database. source = source_cache.get(reply.source_uuid) if not source: logger.error(f"No source found for reply {reply.uuid}") continue nr = Reply(uuid=reply.uuid, journalist_id=user.id, source_id=source.id, filename=reply.filename, size=reply.size) session.add(nr) # All replies fetched from the server have succeeded in being sent, # so we should delete the corresponding draft locally if it exists. try: draft_reply_db_object = session.query(DraftReply).filter_by( uuid=reply.uuid).one() update_draft_replies(session, draft_reply_db_object.source.id, draft_reply_db_object.timestamp, draft_reply_db_object.file_counter, nr.file_counter, commit=False) session.delete(draft_reply_db_object) except NoResultFound: pass # No draft locally stored corresponding to this reply. logger.debug('Added new reply {}'.format(reply.uuid)) # The uuids remaining in local_uuids do not exist on the remote server, so # delete the related records. for deleted_reply in local_replies_by_uuid.values(): delete_single_submission_or_reply_on_disk(deleted_reply, data_dir) session.delete(deleted_reply) logger.debug('Deleted reply {}'.format(deleted_reply.uuid)) session.commit()