def test_server_upload_across_batches(self, mock_upload_comments,
mock_upload_messages,
mock_fill_message_directory):
archive_index = ArchiveMessageIndex(MessageDao())
messages = archive_index.update('test_data')
first_batch = messages[0:6]
second_batch = messages[6:]
mock_fill_message_directory.return_value = ''
# declaring mock objects here because I want to use the ArchiveMessageIndex functionality to build the test data
with mock.patch.object(ArchiveMessageIndex,
'update') as mock_update, mock.patch.object(
MessageDao, 'get') as mock_get:
mock_update.side_effect = [first_batch, second_batch]
mock_get.side_effect = [None, None, messages[6], messages[7]]
server = Server()
server.update_convert_upload()
mock_upload_messages.assert_called_with(
[messages[2].id, messages[3].id])
mock_upload_comments.assert_called_with(set())
server.update_convert_upload()
mock_upload_messages.assert_called_with(
[messages[6].id, messages[7].id])
mock_upload_comments.assert_called_with(set())
def test_split_parent_and_reply_messages(self):
archive_index = ArchiveMessageIndex(MessageDao())
messages = archive_index.update('test_data')
parents, replies = Server.split_parent_and_reply_messages(messages)
self.assertEqual(len(parents), 2)
self.assertEqual(len(replies), 6)
expected_parents = [
'[PATCH v2 1/2] Input: i8042 - Prevent intermixing i8042 commands',
'[PATCH v2 0/4] kselftests/arm64: add PAuth tests'
]
expected_replies = [
'Re: [PATCH] Remove final reference to superfluous smp_commence().',
'[PATCH v2 1/3] dmaengine: add dma_get_channel_caps()',
'[PATCH v2 1/4] kselftests/arm64: add a basic Pointer Authentication test',
'[PATCH v2 2/4] kselftests/arm64: add nop checks for PAuth tests',
'[PATCH v2 3/4] kselftests/arm64: add PAuth test for whether exec() changes keys',
'[PATCH v2 4/4] kselftests/arm64: add PAuth tests for single threaded consistency and key uniqueness'
]
def compare_message_subject(messages: List[Message],
subjects: List[str]):
for message, subject in zip(messages, subjects):
self.assertEqual(message.subject, subject)
compare_message_subject(parents, expected_parents)
compare_message_subject(replies, expected_replies)
def test_update_return_proper_patches(self):
archive_index = ArchiveMessageIndex(MessageDao())
new_messages = archive_index.update(test_data_path())
self.assertEqual(len(new_messages), 8)
subjects = ['Re: [PATCH] Remove final reference to superfluous smp_commence().',
'[PATCH v2 1/3] dmaengine: add dma_get_channel_caps()',
'[PATCH v2 1/2] Input: i8042 - Prevent intermixing i8042 commands',
'[PATCH v2 0/4] kselftests/arm64: add PAuth tests',
'[PATCH v2 1/4] kselftests/arm64: add a basic Pointer Authentication test',
'[PATCH v2 2/4] kselftests/arm64: add nop checks for PAuth tests',
'[PATCH v2 3/4] kselftests/arm64: add PAuth test for whether exec() changes keys',
'[PATCH v2 4/4] kselftests/arm64: add PAuth tests for single threaded consistency and key uniqueness']
compare_message_subjects(self, new_messages, subjects)
def __init__(self) -> None:
rest = gerrit.get_gerrit_rest_api(COOKIE_JAR_PATH, GERRIT_URL)
self.gerrit = gerrit.Gerrit(rest)
self.gerrit_git = git.GerritGit(
git_dir='gerrit_git_dir',
cookie_jar_path=COOKIE_JAR_PATH,
url=GOB_URL,
project='linux/kernel/git/torvalds/linux',
branch='master')
self.message_dao = message_dao.MessageDao()
self.archive_index = ArchiveMessageIndex(self.message_dao)
self.last_hash = self.message_dao.get_last_hash()
archive_updater.setup_archive(GIT_PATH)
os.makedirs(FILE_DIR, exist_ok=True)
os.makedirs(LOG_PATH, exist_ok=True)
logging.get_absl_handler().use_absl_log_file('server_logs', LOG_PATH)
def test_parse_comments_for_single_email_thread(self):
archive_index = ArchiveMessageIndex(MessageDao())
archive_index.update('test_data')
patchset = parse_comments(
archive_index.find(
'<20200827144112.v2.1.I6981f9a9f0c12e60f8038f3b574184f8ffc1b9b5@changeid>'
))
self.assertTrue(len(patchset.patches) > 0)
first_patch = patchset.patches[0]
self.assertEqual(first_patch.set_index, 0)
self.assertNotEqual(first_patch.text, '')
self.assertIn(
'[PATCH v2 1/2] Input: i8042 - Prevent intermixing i8042 commands',
first_patch.text_with_headers)
self.assertEqual(first_patch.comments, [])
def main():
gerrit_url = 'https://linux-review.googlesource.com'
gob_url = 'http://linux.googlesource.com'
rest = get_gerrit_rest_api('gerritcookies', gerrit_url)
gerrit = Gerrit(rest)
gerrit_git = GerritGit(git_dir='gerrit_git_dir',
cookie_jar_path='gerritcookies',
url=gob_url,
project='linux/kernel/git/torvalds/linux',
branch='master')
archive_index = ArchiveMessageIndex(MessageDao())
archive_index.update('test_data')
patchset = parse_comments(
archive_index.find('<*****@*****.**>'))
gerrit_git.apply_patchset_and_cleanup(patchset)
find_and_label_all_revision_ids(gerrit, patchset)
upload_all_comments(gerrit, patchset)
def test_parse_comments_for_multi_email_thread_with_cover_letter(self):
archive_index = ArchiveMessageIndex(MessageDao())
archive_index.update('test_data')
patchset = parse_comments(
archive_index.find(
'<*****@*****.**>'))
self.assertEqual(len(patchset.patches), 4)
first_patch = patchset.patches[0]
self.assertEqual(first_patch.set_index, 1)
self.assertIn(
'[PATCH v2 1/4] kselftests/arm64: add a basic Pointer Authentication test',
first_patch.text_with_headers)
self.assertNotEqual(first_patch.text, '')
self.assertEqual(first_patch.comments, [])
self.assertIn(
'[PATCH v2 2/4] kselftests/arm64: add nop checks for PAuth tests',
patchset.patches[1].text_with_headers)
def test_parse_with_replies(self):
archive_index = ArchiveMessageIndex(MessageDao())
archive_index.update(test_data_path('fake_patch_with_replies/'))
self.assertEqual(archive_index.size(), 2)
patchset = parse_comments(archive_index.find('<patch-message-id>'))
map_comments_to_gerrit(patchset)
self.assertEqual(len(patchset.patches), 1)
patch = patchset.patches[0]
self.compareCommentsPartialMatch(
patch.comments,
[
# TODO: stop treating this as a comment
Comment(
raw_line=-1,
file='',
line=-1,
message=
'On Mon, 31 Aug 2020 at 12:04:46 +0100, The Sender wrote:'
),
Comment(
raw_line=18,
file='file',
line=7, # TODO: should be 5
message='Comment on old line 5, want on line 5 in new file.'
),
Comment(
raw_line=20,
file='file',
line=9, # TODO: should be 7
message='Comment on old line 7, want on line 8 in new file.'
),
])
class Server(object):
def __init__(self) -> None:
rest = gerrit.get_gerrit_rest_api(COOKIE_JAR_PATH, GERRIT_URL)
self.gerrit = gerrit.Gerrit(rest)
self.gerrit_git = git.GerritGit(
git_dir='gerrit_git_dir',
cookie_jar_path=COOKIE_JAR_PATH,
url=GOB_URL,
project='linux/kernel/git/torvalds/linux',
branch='master')
self.message_dao = message_dao.MessageDao()
self.archive_index = ArchiveMessageIndex(self.message_dao)
self.last_hash = self.message_dao.get_last_hash()
archive_updater.setup_archive(GIT_PATH)
os.makedirs(FILE_DIR, exist_ok=True)
os.makedirs(LOG_PATH, exist_ok=True)
logging.get_absl_handler().use_absl_log_file('server_logs', LOG_PATH)
@staticmethod
def remove_files(file_dir: str):
files = glob.glob(f'{file_dir}/*')
for f in files:
os.remove(f)
@staticmethod
def split_parent_and_reply_messages(
messages: List[Message]) -> Tuple[List[Message], List[Message]]:
''' Splits a list of messages into parent (first email in a thread) and replies. '''
parents: List[Message] = []
replies: List[Message] = []
for message in messages:
if not message.in_reply_to:
parents.append(message)
continue
replies.append(message)
return (parents, replies)
def run(self) -> None:
while True:
self.update_convert_upload()
time.sleep(WAIT_TIME)
def update_convert_upload(self) -> None:
new_messages = self.update_message_dir()
# Differentiate between messages to upload and comments
messages_to_upload: List[str] = []
messages_with_new_comments: Set[str] = set()
parent_patches: Set[str] = set()
# First separate between parents and replies. All parents of patchsets will be uploaded
parents, replies = self.split_parent_and_reply_messages(new_messages)
for message in parents:
parent_patches.add(message.id)
messages_to_upload.append(message.id)
# Determine which of the replies should be uploaded
for message in replies:
if message.in_reply_to in parent_patches:
continue
if not self.message_dao.get(message.in_reply_to):
continue
# Reply is a patch to be uploaded (as the parent of patchset is not in new_messages)
if message.is_patch():
messages_to_upload.append(message.id)
# Reply is a comment that's parent is not in this batch of messages. Its parent's comments should be reuploaded
else:
messages_with_new_comments.add(message.in_reply_to)
self.upload_messages(messages_to_upload)
self.upload_comments(messages_with_new_comments)
self.remove_files(FILE_DIR)
def update_message_dir(self) -> List[Message]:
self.last_hash = archive_updater.fill_message_directory(
GIT_PATH, FILE_DIR, self.last_hash)
messages = self.archive_index.update(FILE_DIR)
return messages
def upload_messages(self, messages_to_upload: List[str]):
failed = 0
for message_id in messages_to_upload:
email_thread: Message
try:
email_thread = self.archive_index.find(message_id)
patchset = patch_parser.parse_comments(email_thread)
self.gerrit_git.apply_patchset_and_cleanup(
patchset, self.message_dao)
gerrit.find_and_label_all_revision_ids(self.gerrit, patchset)
gerrit.upload_all_comments(self.gerrit, patchset)
except Exception as e:
failed += 1
failed_message = message_id
if email_thread:
failed_message = email_thread.debug_info()
logging.exception('Failed to upload %s.', failed_message)
continue
if failed > 0:
logging.warning('Failed to upload %d/%d messages', failed,
len(messages_to_upload))
def upload_comments(self, messages_with_new_comments: Collection[str]):
failed = 0
for message_id in messages_with_new_comments:
email_thread: Message
try:
email_thread = self.archive_index.find(message_id)
patchset = patch_parser.parse_comments(email_thread)
gerrit.upload_all_comments(self.gerrit, patchset)
except Exception as e:
failed += 1
failed_message = message_id
if email_thread:
failed_message = email_thread.debug_info()
logging.exception('Failed to upload comments for %s.',
failed_message)
continue
if failed > 0:
logging.warning('Failed to upload %d/%d comments', failed,
len(messages_with_new_comments))
def test_update_with_no_changes_to_data(self):
archive_index = ArchiveMessageIndex(MessageDao())
archive_index.update('test_data')
old_size = archive_index.size()
archive_index.update('test_data')
self.assertEqual(old_size, archive_index.size())