def test_server_upload_across_batches(self, mock_upload_comments, mock_upload_messages, mock_fill_message_directory): archive_index = ArchiveMessageIndex(MessageDao()) messages = archive_index.update('test_data') first_batch = messages[0:6] second_batch = messages[6:] mock_fill_message_directory.return_value = '' # declaring mock objects here because I want to use the ArchiveMessageIndex functionality to build the test data with mock.patch.object(ArchiveMessageIndex, 'update') as mock_update, mock.patch.object( MessageDao, 'get') as mock_get: mock_update.side_effect = [first_batch, second_batch] mock_get.side_effect = [None, None, messages[6], messages[7]] server = Server() server.update_convert_upload() mock_upload_messages.assert_called_with( [messages[2].id, messages[3].id]) mock_upload_comments.assert_called_with(set()) server.update_convert_upload() mock_upload_messages.assert_called_with( [messages[6].id, messages[7].id]) mock_upload_comments.assert_called_with(set())
def test_split_parent_and_reply_messages(self): archive_index = ArchiveMessageIndex(MessageDao()) messages = archive_index.update('test_data') parents, replies = Server.split_parent_and_reply_messages(messages) self.assertEqual(len(parents), 2) self.assertEqual(len(replies), 6) expected_parents = [ '[PATCH v2 1/2] Input: i8042 - Prevent intermixing i8042 commands', '[PATCH v2 0/4] kselftests/arm64: add PAuth tests' ] expected_replies = [ 'Re: [PATCH] Remove final reference to superfluous smp_commence().', '[PATCH v2 1/3] dmaengine: add dma_get_channel_caps()', '[PATCH v2 1/4] kselftests/arm64: add a basic Pointer Authentication test', '[PATCH v2 2/4] kselftests/arm64: add nop checks for PAuth tests', '[PATCH v2 3/4] kselftests/arm64: add PAuth test for whether exec() changes keys', '[PATCH v2 4/4] kselftests/arm64: add PAuth tests for single threaded consistency and key uniqueness' ] def compare_message_subject(messages: List[Message], subjects: List[str]): for message, subject in zip(messages, subjects): self.assertEqual(message.subject, subject) compare_message_subject(parents, expected_parents) compare_message_subject(replies, expected_replies)
def test_parse_comments_for_single_email_thread(self): archive_index = ArchiveMessageIndex(MessageDao()) archive_index.update('test_data') patchset = parse_comments( archive_index.find( '<20200827144112.v2.1.I6981f9a9f0c12e60f8038f3b574184f8ffc1b9b5@changeid>' )) self.assertTrue(len(patchset.patches) > 0) first_patch = patchset.patches[0] self.assertEqual(first_patch.set_index, 0) self.assertNotEqual(first_patch.text, '') self.assertIn( '[PATCH v2 1/2] Input: i8042 - Prevent intermixing i8042 commands', first_patch.text_with_headers) self.assertEqual(first_patch.comments, [])
def main(): gerrit_url = 'https://linux-review.googlesource.com' gob_url = 'http://linux.googlesource.com' rest = get_gerrit_rest_api('gerritcookies', gerrit_url) gerrit = Gerrit(rest) gerrit_git = GerritGit(git_dir='gerrit_git_dir', cookie_jar_path='gerritcookies', url=gob_url, project='linux/kernel/git/torvalds/linux', branch='master') archive_index = ArchiveMessageIndex(MessageDao()) archive_index.update('test_data') patchset = parse_comments( archive_index.find('<*****@*****.**>')) gerrit_git.apply_patchset_and_cleanup(patchset) find_and_label_all_revision_ids(gerrit, patchset) upload_all_comments(gerrit, patchset)
def test_parse_comments_for_multi_email_thread_with_cover_letter(self): archive_index = ArchiveMessageIndex(MessageDao()) archive_index.update('test_data') patchset = parse_comments( archive_index.find( '<*****@*****.**>')) self.assertEqual(len(patchset.patches), 4) first_patch = patchset.patches[0] self.assertEqual(first_patch.set_index, 1) self.assertIn( '[PATCH v2 1/4] kselftests/arm64: add a basic Pointer Authentication test', first_patch.text_with_headers) self.assertNotEqual(first_patch.text, '') self.assertEqual(first_patch.comments, []) self.assertIn( '[PATCH v2 2/4] kselftests/arm64: add nop checks for PAuth tests', patchset.patches[1].text_with_headers)
def test_update_return_proper_patches(self): archive_index = ArchiveMessageIndex(MessageDao()) new_messages = archive_index.update(test_data_path()) self.assertEqual(len(new_messages), 8) subjects = ['Re: [PATCH] Remove final reference to superfluous smp_commence().', '[PATCH v2 1/3] dmaengine: add dma_get_channel_caps()', '[PATCH v2 1/2] Input: i8042 - Prevent intermixing i8042 commands', '[PATCH v2 0/4] kselftests/arm64: add PAuth tests', '[PATCH v2 1/4] kselftests/arm64: add a basic Pointer Authentication test', '[PATCH v2 2/4] kselftests/arm64: add nop checks for PAuth tests', '[PATCH v2 3/4] kselftests/arm64: add PAuth test for whether exec() changes keys', '[PATCH v2 4/4] kselftests/arm64: add PAuth tests for single threaded consistency and key uniqueness'] compare_message_subjects(self, new_messages, subjects)
def test_parse_with_replies(self): archive_index = ArchiveMessageIndex(MessageDao()) archive_index.update(test_data_path('fake_patch_with_replies/')) self.assertEqual(archive_index.size(), 2) patchset = parse_comments(archive_index.find('<patch-message-id>')) map_comments_to_gerrit(patchset) self.assertEqual(len(patchset.patches), 1) patch = patchset.patches[0] self.compareCommentsPartialMatch( patch.comments, [ # TODO: stop treating this as a comment Comment( raw_line=-1, file='', line=-1, message= 'On Mon, 31 Aug 2020 at 12:04:46 +0100, The Sender wrote:' ), Comment( raw_line=18, file='file', line=7, # TODO: should be 5 message='Comment on old line 5, want on line 5 in new file.' ), Comment( raw_line=20, file='file', line=9, # TODO: should be 7 message='Comment on old line 7, want on line 8 in new file.' ), ])
class Server(object): def __init__(self) -> None: rest = gerrit.get_gerrit_rest_api(COOKIE_JAR_PATH, GERRIT_URL) self.gerrit = gerrit.Gerrit(rest) self.gerrit_git = git.GerritGit( git_dir='gerrit_git_dir', cookie_jar_path=COOKIE_JAR_PATH, url=GOB_URL, project='linux/kernel/git/torvalds/linux', branch='master') self.message_dao = message_dao.MessageDao() self.archive_index = ArchiveMessageIndex(self.message_dao) self.last_hash = self.message_dao.get_last_hash() archive_updater.setup_archive(GIT_PATH) os.makedirs(FILE_DIR, exist_ok=True) os.makedirs(LOG_PATH, exist_ok=True) logging.get_absl_handler().use_absl_log_file('server_logs', LOG_PATH) @staticmethod def remove_files(file_dir: str): files = glob.glob(f'{file_dir}/*') for f in files: os.remove(f) @staticmethod def split_parent_and_reply_messages( messages: List[Message]) -> Tuple[List[Message], List[Message]]: ''' Splits a list of messages into parent (first email in a thread) and replies. ''' parents: List[Message] = [] replies: List[Message] = [] for message in messages: if not message.in_reply_to: parents.append(message) continue replies.append(message) return (parents, replies) def run(self) -> None: while True: self.update_convert_upload() time.sleep(WAIT_TIME) def update_convert_upload(self) -> None: new_messages = self.update_message_dir() # Differentiate between messages to upload and comments messages_to_upload: List[str] = [] messages_with_new_comments: Set[str] = set() parent_patches: Set[str] = set() # First separate between parents and replies. All parents of patchsets will be uploaded parents, replies = self.split_parent_and_reply_messages(new_messages) for message in parents: parent_patches.add(message.id) messages_to_upload.append(message.id) # Determine which of the replies should be uploaded for message in replies: if message.in_reply_to in parent_patches: continue if not self.message_dao.get(message.in_reply_to): continue # Reply is a patch to be uploaded (as the parent of patchset is not in new_messages) if message.is_patch(): messages_to_upload.append(message.id) # Reply is a comment that's parent is not in this batch of messages. Its parent's comments should be reuploaded else: messages_with_new_comments.add(message.in_reply_to) self.upload_messages(messages_to_upload) self.upload_comments(messages_with_new_comments) self.remove_files(FILE_DIR) def update_message_dir(self) -> List[Message]: self.last_hash = archive_updater.fill_message_directory( GIT_PATH, FILE_DIR, self.last_hash) messages = self.archive_index.update(FILE_DIR) return messages def upload_messages(self, messages_to_upload: List[str]): failed = 0 for message_id in messages_to_upload: email_thread: Message try: email_thread = self.archive_index.find(message_id) patchset = patch_parser.parse_comments(email_thread) self.gerrit_git.apply_patchset_and_cleanup( patchset, self.message_dao) gerrit.find_and_label_all_revision_ids(self.gerrit, patchset) gerrit.upload_all_comments(self.gerrit, patchset) except Exception as e: failed += 1 failed_message = message_id if email_thread: failed_message = email_thread.debug_info() logging.exception('Failed to upload %s.', failed_message) continue if failed > 0: logging.warning('Failed to upload %d/%d messages', failed, len(messages_to_upload)) def upload_comments(self, messages_with_new_comments: Collection[str]): failed = 0 for message_id in messages_with_new_comments: email_thread: Message try: email_thread = self.archive_index.find(message_id) patchset = patch_parser.parse_comments(email_thread) gerrit.upload_all_comments(self.gerrit, patchset) except Exception as e: failed += 1 failed_message = message_id if email_thread: failed_message = email_thread.debug_info() logging.exception('Failed to upload comments for %s.', failed_message) continue if failed > 0: logging.warning('Failed to upload %d/%d comments', failed, len(messages_with_new_comments))
def test_update_with_no_changes_to_data(self): archive_index = ArchiveMessageIndex(MessageDao()) archive_index.update('test_data') old_size = archive_index.size() archive_index.update('test_data') self.assertEqual(old_size, archive_index.size())