def create_comment(**kwargs): """Create 'Comment' object.""" values = { 'submitter': create_person(), 'submission': create_patch(), 'msgid': make_msgid(), 'content': SAMPLE_CONTENT, } values.update(kwargs) comment = Comment(**values) comment.save() return comment
def setUp(self): defaults.project.save() self.person = defaults.patch_author_person self.person.save() self.patch = Patch(project=defaults.project, msgid='p1', name='testpatch', submitter=self.person, diff='', content='comment 1 text\nAcked-by: 1\n') self.patch.save() comment = Comment(submission=self.patch, msgid='p2', submitter=self.person, content='comment 2 text\nAcked-by: 2\n') comment.save()
def setUp(self): defaults.project.save() self.person = defaults.patch_author_person self.person.save() self.patch = Patch(project = defaults.project, msgid = 'p1', name = 'testpatch', submitter = self.person, content = '') self.patch.save() self.txt = 'some comment\n---\n some/file | 1 +\n' comment = Comment(patch = self.patch, msgid = 'p1', submitter = self.person, content = self.txt) comment.save()
def parse_mail(mail): # some basic sanity checks if 'From' not in mail: LOGGER.debug("Ignoring mail due to missing 'From'") return 1 if 'Subject' not in mail: LOGGER.debug("Ignoring mail due to missing 'Subject'") return 1 if 'Message-Id' not in mail: LOGGER.debug("Ignoring mail due to missing 'Message-Id'") return 1 hint = mail.get('X-Patchwork-Hint', '').lower() if hint == 'ignore': LOGGER.debug("Ignoring mail due to 'ignore' hint") return 0 project = find_project(mail) if project is None: LOGGER.error('Failed to find a project for mail') return 1 (author, save_required) = find_author(mail) content = find_content(project, mail) if not content: return 0 patch = content.patch comment = content.comment series = content.series revision = content.revision msgid = (content.msgid or mail.get('Message-Id').strip()) series_revision_complete.connect(on_revision_complete) if series: if save_required: author.save() save_required = False series.project = project series.submitter = author series.save() LOGGER.debug('Series saved') if revision: revision.series = series revision.save() LOGGER.debug('Revision saved') if patch: delegate = get_delegate(mail.get('X-Patchwork-Delegate', '').strip()) if not delegate: delegate = auto_delegate(project, content.filenames) # we delay the saving until we know we have a patch. if save_required: author.save() save_required = False patch.submitter = author patch.msgid = msgid patch.project = project patch.state = get_state(mail.get('X-Patchwork-State', '').strip()) patch.delegate = delegate patch.save() if revision: revision.add_patch(patch, content.patch_order) LOGGER.debug('Patch saved') if comment: if save_required: author.save() # we defer this assignment until we know that we have a saved patch if patch: comment.patch = patch comment.submitter = author comment.msgid = msgid comment.save() LOGGER.debug('Comment saved') # if comment's author has project-maintainer permissions, # parse comment content and process the status-change command # if it is found if author.user and project in \ (author.user).profile.maintainer_projects.all(): cmd = None comment_re = re.compile('^\[Patchwork-Status:\s*(Under Review|\ Rejected|RFC|Not Applicable|Changes Requested|Awaiting Upstream|Superseded|\ Deferred)\]$', re.M | re.I) # if multiple valid status-change commands are found, use last one for match in comment_re.finditer(comment.content): cmd = re.sub(r'(\[Patchwork-Status:\s*)(.*)(\])', r'\2', "{}".format(match.group(0))) if cmd is not None: new_state = State.objects.get(name=cmd.title()) mod_patch = Patch.objects.get(pk=comment.patch.pk) if new_state and mod_patch.state != new_state: mod_patch.state = new_state mod_patch.save() cmd_message = 'This is a system generated Comment: Patch \ %s status was updated to "%s" due to request in comment.' % ( comment.patch.pk, cmd) cmd_msgid = "%s: System generated by comment %s" % ( datetime.datetime.now().strftime("\ %d%b%Y.%H:%M:%S.%f"), comment.pk) new_comment = Comment(pk=None, patch=comment.patch, content=cmd_message, date=datetime.datetime.now(), submitter=comment.submitter, msgid=cmd_msgid) new_comment.save() series_revision_complete.disconnect(on_revision_complete) return 0
def create_tag_comment(self, patch, tagtype = None): comment = Comment(patch=patch, msgid=str(datetime.datetime.now()), submitter=defaults.patch_author_person, content=self.create_tag(tagtype)) comment.save() return comment
def parse_mail(mail, list_id=None): """Parse a mail and add to the database. Args: mail (`mbox.Mail`): Mail to parse and add. list_id (str): Mailing list ID Returns: None """ # some basic sanity checks if 'From' not in mail: raise ValueError("Missing 'From' header") if 'Subject' not in mail: raise ValueError("Missing 'Subject' header") if 'Message-Id' not in mail: raise ValueError("Missing 'Message-Id' header") hint = clean_header(mail.get('X-Patchwork-Hint', '')) if hint and hint.lower() == 'ignore': logger.debug("Ignoring email due to 'ignore' hint") return if list_id: project = find_project_by_id(list_id) else: project = find_project_by_header(mail) if project is None: logger.error('Failed to find a project for email') return # parse metadata msgid = clean_header(mail.get('Message-Id')) if not msgid: raise ValueError("Broken 'Message-Id' header") msgid = msgid[:255] author = find_author(mail) subject = mail.get('Subject') name, prefixes = clean_subject(subject, [project.linkname]) is_comment = subject_check(subject) x, n = parse_series_marker(prefixes) version = parse_version(name, prefixes) refs = find_references(mail) date = find_date(mail) headers = find_headers(mail) # parse content if not is_comment: diff, message = find_patch_content(mail) else: diff, message = find_comment_content(mail) if not (diff or message): return # nothing to work with pull_url = parse_pull_request(message) # build objects if not is_comment and (diff or pull_url): # patches or pull requests # we delay the saving until we know we have a patch. author.save() delegate = find_delegate_by_header(mail) if not delegate and diff: filenames = find_filenames(diff) delegate = find_delegate_by_filename(project, filenames) # if we don't have a series marker, we will never have an existing # series to match against. series = None if n: series = find_series(project, mail) else: x = n = 1 # We will create a new series if: # - there is no existing series to assign this patch to, or # - there is an existing series, but it already has a patch with this # number in it if not series or ( SeriesPatch.objects.filter(series=series, number=x).count()): series = Series(project=project, date=date, submitter=author, version=version, total=n) series.save() # NOTE(stephenfin) We must save references for series. We # do this to handle the case where a later patch is # received first. Without storing references, it would not # be possible to identify the relationship between patches # as the earlier patch does not reference the later one. for ref in refs + [msgid]: ref = ref[:255] # we don't want duplicates try: # we could have a ref to a previous series. (For # example, a series sent in reply to another # series.) That should not create a series ref # for this series, so check for the msg-id only, # not the msg-id/series pair. SeriesReference.objects.get(msgid=ref, series__project=project) except SeriesReference.DoesNotExist: SeriesReference.objects.create(series=series, msgid=ref) patch = Patch( msgid=msgid, project=project, name=name[:255], date=date, headers=headers, submitter=author, content=message, diff=diff, pull_url=pull_url, delegate=delegate, state=find_state(mail)) patch.save() logger.debug('Patch saved') # add to a series if we have found one, and we have a numbered # patch. Don't add unnumbered patches (for example diffs sent # in reply, or just messages with random refs/in-reply-tos) if series and x: series.add_patch(patch, x) return patch elif x == 0: # (potential) cover letters # if refs are empty, it's implicitly a cover letter. If not, # however, we need to see if a match already exists and, if # not, assume that it is indeed a new cover letter is_cover_letter = False if not is_comment: if not refs == []: try: CoverLetter.objects.all().get(name=name) except CoverLetter.DoesNotExist: # if no match, this is a new cover letter is_cover_letter = True except CoverLetter.MultipleObjectsReturned: # if multiple cover letters are found, just ignore pass else: is_cover_letter = True if is_cover_letter: author.save() # we don't use 'find_series' here as a cover letter will # always be the first item in a thread, thus the references # could only point to a different series or unrelated # message try: series = SeriesReference.objects.get( msgid=msgid, series__project=project).series except SeriesReference.DoesNotExist: series = None if not series: series = Series(project=project, date=date, submitter=author, version=version, total=n) series.save() # we don't save the in-reply-to or references fields # for a cover letter, as they can't refer to the same # series SeriesReference.objects.get_or_create(series=series, msgid=msgid) cover_letter = CoverLetter( msgid=msgid, project=project, name=name[:255], date=date, headers=headers, submitter=author, content=message) cover_letter.save() logger.debug('Cover letter saved') series.add_cover_letter(cover_letter) return cover_letter # comments # we only save comments if we have the parent email submission = find_submission_for_comment(project, refs) if not submission: return author.save() comment = Comment( submission=submission, msgid=msgid, date=date, headers=headers, submitter=author, content=message) comment.save() logger.debug('Comment saved') return comment
def parse_mail(mail, list_id=None): """Parse a mail and add to the database. Args: mail (`mbox.Mail`): Mail to parse and add. list_id (str): Mailing list ID Returns: None """ # some basic sanity checks if 'From' not in mail: raise ValueError("Missing 'From' header") if 'Subject' not in mail: raise ValueError("Missing 'Subject' header") if 'Message-Id' not in mail: raise ValueError("Missing 'Message-Id' header") hint = mail.get('X-Patchwork-Hint', '').lower() if hint == 'ignore': LOGGER.debug("Ignoring email due to 'ignore' hint") return if list_id: project = find_project_by_id(list_id) else: project = find_project_by_header(mail) if project is None: LOGGER.error('Failed to find a project for email') return # parse content diff, message = find_content(project, mail) if not (diff or message): return # nothing to work with msgid = mail.get('Message-Id').strip() author = find_author(mail) name, prefixes = clean_subject(mail.get('Subject'), [project.linkname]) x, n = parse_series_marker(prefixes) refs = find_references(mail) date = find_date(mail) headers = find_headers(mail) pull_url = find_pull_request(message) # build objects if diff or pull_url: # patches or pull requests # we delay the saving until we know we have a patch. author.save() delegate = find_delegate(mail) if not delegate and diff: filenames = patch_get_filenames(diff) delegate = auto_delegate(project, filenames) patch = Patch( msgid=msgid, project=project, name=name, date=date, headers=headers, submitter=author, content=message, diff=diff, pull_url=pull_url, delegate=delegate, state=find_state(mail)) patch.save() LOGGER.debug('Patch saved') return patch elif x == 0: # (potential) cover letters # if refs are empty, it's implicitly a cover letter. If not, # however, we need to see if a match already exists and, if # not, assume that it is indeed a new cover letter is_cover_letter = False if not refs == []: try: CoverLetter.objects.all().get(name=name) except CoverLetter.DoesNotExist: # no match => new cover is_cover_letter = True else: is_cover_letter = True if is_cover_letter: author.save() cover_letter = CoverLetter( msgid=msgid, project=project, name=name, date=date, headers=headers, submitter=author, content=message) cover_letter.save() LOGGER.debug('Cover letter saved') return cover_letter # comments # we only save comments if we have the parent email submission = find_submission_for_comment(project, refs) if not submission: return author.save() comment = Comment( submission=submission, msgid=msgid, date=date, headers=headers, submitter=author, content=message) comment.save() LOGGER.debug('Comment saved') return comment
def parse_mail(mail, list_id=None): """Parse a mail and add to the database. Args: mail (`mbox.Mail`): Mail to parse and add. list_id (str): Mailing list ID Returns: None """ # some basic sanity checks if 'From' not in mail: raise ValueError("Missing 'From' header") if 'Subject' not in mail: raise ValueError("Missing 'Subject' header") if 'Message-Id' not in mail: raise ValueError("Missing 'Message-Id' header") hint = clean_header(mail.get('X-Patchwork-Hint', '')) if hint and hint.lower() == 'ignore': logger.debug("Ignoring email due to 'ignore' hint") return if list_id: project = find_project_by_id(list_id) else: project = find_project_by_header(mail) if project is None: logger.error('Failed to find a project for email') return # parse metadata msgid = clean_header(mail.get('Message-Id')) if not msgid: raise ValueError("Broken 'Message-Id' header") msgid = msgid[:255] author = find_author(mail) subject = mail.get('Subject') name, prefixes = clean_subject(subject, [project.linkname]) is_comment = subject_check(subject) x, n = parse_series_marker(prefixes) version = parse_version(name, prefixes) refs = find_references(mail) date = find_date(mail) headers = find_headers(mail) # parse content if not is_comment: diff, message = find_patch_content(mail) else: diff, message = find_comment_content(mail) if not (diff or message): return # nothing to work with pull_url = parse_pull_request(message) # build objects if not is_comment and (diff or pull_url): # patches or pull requests # we delay the saving until we know we have a patch. author.save() delegate = find_delegate_by_header(mail) if not delegate and diff: filenames = find_filenames(diff) delegate = find_delegate_by_filename(project, filenames) # if we don't have a series marker, we will never have an existing # series to match against. series = None if n: series = find_series(project, mail) else: x = n = 1 # We will create a new series if: # - there is no existing series to assign this patch to, or # - there is an existing series, but it already has a patch with this # number in it if not series or (SeriesPatch.objects.filter(series=series, number=x).count()): series = Series(project=project, date=date, submitter=author, version=version, total=n) series.save() # NOTE(stephenfin) We must save references for series. We # do this to handle the case where a later patch is # received first. Without storing references, it would not # be possible to identify the relationship between patches # as the earlier patch does not reference the later one. for ref in refs + [msgid]: ref = ref[:255] # we don't want duplicates try: # we could have a ref to a previous series. (For # example, a series sent in reply to another # series.) That should not create a series ref # for this series, so check for the msg-id only, # not the msg-id/series pair. SeriesReference.objects.get(msgid=ref, series__project=project) except SeriesReference.DoesNotExist: SeriesReference.objects.create(series=series, msgid=ref) patch = Patch(msgid=msgid, project=project, name=name[:255], date=date, headers=headers, submitter=author, content=message, diff=diff, pull_url=pull_url, delegate=delegate, state=find_state(mail)) patch.save() logger.debug('Patch saved') # add to a series if we have found one, and we have a numbered # patch. Don't add unnumbered patches (for example diffs sent # in reply, or just messages with random refs/in-reply-tos) if series and x: series.add_patch(patch, x) return patch elif x == 0: # (potential) cover letters # if refs are empty, it's implicitly a cover letter. If not, # however, we need to see if a match already exists and, if # not, assume that it is indeed a new cover letter is_cover_letter = False if not is_comment: if not refs == []: try: CoverLetter.objects.all().get(name=name) except CoverLetter.DoesNotExist: # if no match, this is a new cover letter is_cover_letter = True except CoverLetter.MultipleObjectsReturned: # if multiple cover letters are found, just ignore pass else: is_cover_letter = True if is_cover_letter: author.save() # we don't use 'find_series' here as a cover letter will # always be the first item in a thread, thus the references # could only point to a different series or unrelated # message try: series = SeriesReference.objects.get( msgid=msgid, series__project=project).series except SeriesReference.DoesNotExist: series = None if not series: series = Series(project=project, date=date, submitter=author, version=version, total=n) series.save() # we don't save the in-reply-to or references fields # for a cover letter, as they can't refer to the same # series SeriesReference.objects.get_or_create(series=series, msgid=msgid) cover_letter = CoverLetter(msgid=msgid, project=project, name=name[:255], date=date, headers=headers, submitter=author, content=message) cover_letter.save() logger.debug('Cover letter saved') series.add_cover_letter(cover_letter) return cover_letter # comments # we only save comments if we have the parent email submission = find_submission_for_comment(project, refs) if not submission: return author.save() comment = Comment(submission=submission, msgid=msgid, date=date, headers=headers, submitter=author, content=message) comment.save() logger.debug('Comment saved') return comment
def create_tag_comment(self, patch, tagtype=None): comment = Comment(patch=patch, msgid=str(datetime.datetime.now()), submitter=defaults.patch_author_person, content=self.create_tag(tagtype)) comment.save() return comment
def find_content(project, mail, force_comment=False): patchbuf = None commentbuf = '' pullurl = None is_attachment = False for part in mail.walk(): if part.get_content_maintype() != 'text': continue payload = part.get_payload(decode=True) subtype = part.get_content_subtype() if not isinstance(payload, six.text_type): charset = part.get_content_charset() # Check that we have a charset that we understand. Otherwise, # ignore it and fallback to our standard set. if charset is not None: try: codecs.lookup(charset) except LookupError: charset = None # If there is no charset or if it is unknown, then try some common # charsets before we fail. if charset is None: try_charsets = ['utf-8', 'windows-1252', 'iso-8859-1'] else: try_charsets = [charset] for cset in try_charsets: decoded_payload = try_decode(payload, cset) if decoded_payload is not None: break payload = decoded_payload # Could not find a valid decoded payload. Fail. if payload is None: return None if subtype in ['x-patch', 'x-diff']: is_attachment = True patchbuf = payload elif subtype == 'plain': c = payload if not patchbuf: (patchbuf, c) = parse_patch(payload) if not pullurl: pullurl = find_pull_request(payload) if c is not None: commentbuf += c.strip() + '\n' if force_comment: patchbuf = None ret = MailContent() drop_prefixes = [project.linkname, project.get_listemail_tag()] drop_prefixes += project.get_subject_prefix_tags() (name, prefixes) = clean_subject(mail.get('Subject'), drop_prefixes) (x, n) = parse_series_marker(prefixes) refs = build_references_list(mail) is_root = refs == [] is_cover_letter = is_root and x == 0 is_patch = patchbuf is not None drop_patch = not is_attachment and \ project.git_send_email_only and not is_git_send_email(mail) if pullurl or (is_patch and not drop_patch): ret.patch_order = x or 1 ret.patch = Patch(name=name, pull_url=pullurl, content=patchbuf, date=mail_date(mail), headers=mail_headers(mail)) if patchbuf: ret.filenames = patch_get_filenames(patchbuf) # Create/update the Series and SeriesRevision objects if is_cover_letter or is_patch: msgid = mail.get('Message-Id').strip() # Series get a generic name when they don't start by a cover letter or # when they haven't received the root message yet. Except when it's # only 1 patch, then the series takes the patch subject as name. series_name = None if is_cover_letter or n is None: series_name = strip_prefixes(name) (ret.series, ret.revision, ret.patch_order, n) = \ find_series_for_mail(project, series_name, msgid, is_patch, ret.patch_order, n, refs) ret.revision.n_patches = n or 1 date = mail_date(mail) if not ret.series.submitted or date < ret.series.submitted: ret.series.submitted = date if is_cover_letter: ret.revision.cover_letter = clean_content(commentbuf) return ret if commentbuf: # If this is a new patch, we defer setting comment.patch until # patch has been saved by the caller if ret.patch: ret.comment = Comment(date=mail_date(mail), content=clean_content(commentbuf), headers=mail_headers(mail)) else: cpatch = find_patch_for_comment(project, refs) if not cpatch: return ret ret.comment = Comment(patch=cpatch, date=mail_date(mail), content=clean_content(commentbuf), headers=mail_headers(mail)) # make sure we always have a valid (series,revision) tuple if we have a # patch. We don't consider pull requests a series. if ret.patch and not pullurl and (not ret.series or not ret.revision): raise Exception("Could not find series for: %s" % name) return ret
def parse_mail(mail): # some basic sanity checks if 'From' not in mail: LOGGER.debug("Ignoring mail due to missing 'From'") return 1 if 'Subject' not in mail: LOGGER.debug("Ignoring mail due to missing 'Subject'") return 1 if 'Message-Id' not in mail: LOGGER.debug("Ignoring mail due to missing 'Message-Id'") return 1 hint = mail.get('X-Patchwork-Hint', '').lower() if hint == 'ignore': LOGGER.debug("Ignoring mail due to 'ignore' hint") return 0 project = find_project(mail) if project is None: LOGGER.error('Failed to find a project for mail') return 1 (author, save_required) = find_author(mail) content = find_content(project, mail) if not content: return 0 patch = content.patch comment = content.comment series = content.series revision = content.revision msgid = (content.msgid or mail.get('Message-Id').strip()) series_revision_complete.connect(on_revision_complete) if series: if save_required: author.save() save_required = False series.project = project series.submitter = author series.save() LOGGER.debug('Series saved') if revision: revision.series = series revision.save() LOGGER.debug('Revision saved') if patch: delegate = get_delegate(mail.get('X-Patchwork-Delegate', '').strip()) if not delegate: delegate = auto_delegate(project, content.filenames) # we delay the saving until we know we have a patch. if save_required: author.save() save_required = False patch.submitter = author patch.msgid = msgid patch.project = project patch.state = get_state(mail.get('X-Patchwork-State', '').strip()) patch.delegate = delegate patch.save() if revision: revision.add_patch(patch, content.patch_order) LOGGER.debug('Patch saved') if comment: if save_required: author.save() # we defer this assignment until we know that we have a saved patch if patch: comment.patch = patch comment.submitter = author comment.msgid = msgid comment.save() LOGGER.debug('Comment saved') # look for a status-change command string in the comment cmd = None comment_re = re.compile( '^(?:\n|\r\n?)\[Patchwork-Status:\s*\ (Under Review|Rejected|RFC|Not Applicable|Changes Requested|Awaiting \ Upstream|Superseded|Deferred)\](?:\n|\r\n?)$', re.M | re.I) # if multiple valid status-change commands are found, use last one for match in comment_re.finditer(comment.content): cmd = re.sub( r'((?:\n|\r\n?)\[Patchwork-Status:\s*)(.*)(\](?:\n|\r\n?))', r'\2', "{}".format(match.group(0))) # if a status-change command string is found, see if comment's author # has either project-maintainer permissions or is the series submitter # and process the command if true. if cmd is not None: refs = build_references_list(mail) if not refs == []: if not series: series = SeriesRevision.objects.filter( series__project=project, root_msgid=refs[-1]).reverse()[0].series if (author.user and project in (author.user).profile.maintainer_projects.all()) or ( author and author == series.submitter): new_state = State.objects.get(name=cmd.title()) mod_patch = Patch.objects.get(pk=comment.patch.pk) if new_state and mod_patch.state != new_state: mod_patch.state = new_state mod_patch.save() cmd_message = 'This is a system generated Comment: \ Patch %s status was updated to "%s"\ndue to request in comment body.' % ( comment.patch.pk, cmd) cmd_msgid = "%s: System generated by comment %s" % ( datetime.datetime.now().strftime("\ %d%b%Y.%H:%M:%S.%f"), comment.pk) new_comment = Comment(pk=None, patch=comment.patch, content=cmd_message, date=datetime.datetime.now(), submitter=comment.submitter, msgid=cmd_msgid) new_comment.save() else: # notify that a patch-status change was attempted without # apropriate submitter/maintainer permissions cmd_message = 'This is a system generated Comment: \ A command to change a patch-status through\nemail was detected in comment, \ but the sender email does not belong either to\na project maintainer or to \ the series submitter, the only approved users for\nthis function. \ Ensure you are using an approved email address when submitting.' cmd_msgid = "%s: System generated by comment %s" % ( datetime.datetime.now().strftime("\ %d%b%Y.%H:%M:%S.%f"), comment.pk) new_comment = Comment(pk=None, patch=comment.patch, content=cmd_message, date=datetime.datetime.now(), submitter=comment.submitter, msgid=cmd_msgid) new_comment.save() series_revision_complete.disconnect(on_revision_complete) return 0