def add_message_from_mbox(self, mbox, user, project_name=None):
def find_message_projects(m):
return [p for p in Project.objects.all() if p.recognizes(m)]
m = MboxMessage(mbox)
msgid = m.get_message_id()
if project_name:
projects = [Project.object.get(name=project_name)]
else:
projects = find_message_projects(m)
for p in projects:
msg = Message(message_id=msgid,
in_reply_to=m.get_in_reply_to() or "",
date=m.get_date(),
subject=m.get_subject(),
stripped_subject=m.get_subject(strip_tags=True),
version=m.get_version(),
sender=m.get_from(),
recipients=m.get_to() + m.get_cc(),
prefixes=m.get_prefixes(),
is_series_head=m.is_series_head(),
is_patch=m.is_patch(),
patch_num=m.get_num()[0])
msg.project = p
if self.filter(message_id=msgid, project__name=p.name).first():
raise self.DuplicateMessageError(msgid)
msg.save_mbox(mbox)
msg.save()
emit_event("MessageAdded", message=msg)
self.update_series(msg)
return projects
def _add_to_queue(self, user, m, queue):
for x in [m] + list(m.get_patches()):
q, created = QueuedSeries.objects.get_or_create(user=user,
message=x,
name=queue)
if created:
emit_event("MessageQueued", user=user, message=x, queue=q)
def add_test_report(self, user, project, tester, test, head,
base, identity, passed, log, is_timeout):
# Find a project or series depending on the test type and assign it to obj
if identity["type"] == "project":
obj = Project.objects.get(name=project)
is_proj_report = True
project = obj.name
elif identity["type"] == "series":
message_id = identity["message-id"]
obj = Message.objects.find_series(message_id, project)
if not obj:
raise Exception("Series doesn't exist")
is_proj_report = False
project = obj.project.name
obj.set_property("testing.report." + test,
{"passed": passed,
"is_timeout": is_timeout,
"user": user.username,
"tester": tester or user.username,
})
obj.set_property("testing.log." + test, log)
if not passed:
obj.set_property("testing.failed", True)
reports = [x for x in obj.get_properties() if x.startswith("testing.report.")]
done_tests = set([x[len("testing.report."):] for x in reports])
all_tests = set([k for k, v in self.get_tests(obj).items() if v["enabled"]])
if all_tests.issubset(done_tests):
obj.set_property("testing.done", True)
if all_tests.issubset(done_tests):
obj.set_property("testing.tested-head", head)
emit_event("TestingReport", tester=tester, user=user.username,
obj=obj, passed=passed, test=test, log=log,
is_timeout=is_timeout)
def set_property(self, prop, value):
old_val = self.get_property(prop)
self._do_set_property(prop, value)
emit_event("SetProperty",
obj=self,
name=prop,
value=value,
old_value=old_val)
def save(self):
self.last_update = datetime.datetime.utcnow()
old_result = Result.objects.filter(pk=self.pk).first()
old_status = old_result.status if old_result else None
r = super(Result, self).save()
emit_event("ResultUpdate",
obj=self.obj,
old_status=old_status,
result=self)
def _drop_from_queue(self, user, m, queue):
query = QueuedSeries.objects.filter(user=user,
message__in=m.get_patches() + [m],
name=queue)
for q in query:
emit_event("MessageDropping",
user=user,
message=q.message,
queue=q)
q.delete()
def _poll_project(self, po):
repo, branch = self._get_project_repo_and_branch(po)
cache_repo = self._update_cache_repo(po.name, repo, branch)
head = subprocess.check_output(["git", "rev-parse", branch],
cwd=cache_repo).decode('utf-8').strip()
old_head = po.get_property("git.head")
if old_head != head:
po.set_property("git.head", head)
po.set_property("git.repo", repo)
emit_event("ProjectGitUpdate", project=po.name)
return cache_repo
def add_test_report(
self,
request,
project,
tester,
test,
head,
base,
identity,
passed,
log,
is_timeout,
):
# Find a project or series depending on the test type and assign it to obj
if identity["type"] == "project":
obj = Project.objects.get(name=project)
project = obj.name
elif identity["type"] == "series":
message_id = identity["message-id"]
obj = Message.objects.find_series(message_id, project)
if not obj:
raise Exception("Series doesn't exist")
project = obj.project.name
user = request.user
r = self.get_testing_result(obj, test)
r.data = {
"is_timeout": is_timeout,
"user": user.username,
"head": head,
"tester": tester or user.username,
}
r.log = log
r.status = Result.SUCCESS if passed else Result.FAILURE
r.save()
log_url = self.reverse_testing_log(obj, test, request=request)
html_log_url = self.reverse_testing_log(obj,
test,
request=request,
html=True)
emit_event(
"TestingReport",
tester=tester,
user=user.username,
obj=obj,
passed=passed,
test=test,
log=log,
log_url=log_url,
html_log_url=html_log_url,
is_timeout=is_timeout,
)
def set_property(self, prop, value):
if value is None:
self.delete_property(prop)
return
x = self.properties
*path, last = prop.split(".")
for item in path:
x = x.setdefault(item, {})
old_val = x.get(last)
x[last] = value
self.save()
emit_event("SetProperty", obj=self, name=prop, value=value, old_value=old_val)
def delete_property(self, prop):
x = self.properties
*path, last = prop.split(".")
for item in path:
if not item in x:
return
x = x[item]
if not last in x:
return
old_val = x[last]
del x[last]
self.save()
emit_event("SetProperty", obj=self, name=prop, value=None, old_value=old_val)
def on_message_added(self, event, message):
series = message.get_series_head()
if not series:
return
def newer_than(m1, m2):
return m1.version > m2.version and m1.date >= m2.date
for m in series.get_alternative_revisions():
if newer_than(m, series):
series.is_obsolete = True
series.save()
series.set_property("obsoleted-by", m.message_id)
elif newer_than(series, m):
m.is_obsolete = True
m.save()
m.set_property("obsoleted-by", series.message_id)
updated = self.update_tags(series)
for p in series.get_patches():
updated = updated or self.update_tags(p)
reviewers = set()
num_reviewed = 0
def _find_reviewers(what):
ret = set()
for rev_tag in [
x for x in what.tags
if x.lower().startswith(REV_BY_PREFIX.lower())
]:
ret.add(parse_address(rev_tag[len(REV_BY_PREFIX):]))
return ret
for p in series.get_patches():
first = True
this_reviewers = _find_reviewers(p)
if this_reviewers:
if first:
num_reviewed += 1
first = False
reviewers = reviewers.union(this_reviewers)
series_reviewers = _find_reviewers(series)
reviewers = reviewers.union(series_reviewers)
if num_reviewed == series.get_num()[1] or series_reviewers:
series.is_reviewed = True
series.save()
series.set_property("reviewers", list(reviewers))
if updated:
emit_event("TagsUpdate", series=series)
def save(self, *args, **kwargs):
self.last_update = datetime.datetime.utcnow()
old_result = Result.objects.filter(pk=self.pk).first()
old_status = old_result.status if old_result else None
old_entry = old_result.log_entry if old_result else None
super(Result, self).save(*args, **kwargs)
if self.log_entry is None and old_entry is not None:
# Quick way to check if the field was actually saved to the database
new_result = Result.objects.filter(pk=self.pk).first()
if new_result.log_entry is None:
old_entry.delete()
emit_event("ResultUpdate", obj=self.obj, old_status=old_status, result=self)
def create(self, project, **validated_data):
mbox = validated_data.pop('mbox')
m = MboxMessage(mbox)
msg = Message(**validated_data)
if 'in_reply_to' not in validated_data:
msg.in_reply_to = m.get_in_reply_to() or ""
msg.stripped_subject = m.get_subject(strip_tags=True)
msg.version = m.get_version()
msg.prefixes = m.get_prefixes()
msg.is_series_head = m.is_series_head()
msg.is_patch = m.is_patch()
msg.patch_num = m.get_num()[0]
msg.project = project
msg.save_mbox(mbox)
msg.save()
emit_event("MessageAdded", message=msg)
self.update_series(msg)
return msg
def on_message_added(self, event, message):
series = message.get_series_head()
if not series:
return
for m in series.get_alternative_revisions():
if m.version > series.version:
series.set_property("obsoleted-by", m.message_id)
elif m.version < series.version:
m.set_property("obsoleted-by", series.message_id)
updated = self.update_tags(series)
for p in series.get_patches():
updated = updated or self.update_tags(p)
reviewers = set()
num_reviewed = 0
def _find_reviewers(what):
ret = set()
for rev_tag in [
x for x in what.get_property("tags", [])
if x.lower().startswith(REV_BY_PREFIX.lower())
]:
ret.add(parse_address(rev_tag[len(REV_BY_PREFIX):]))
return ret
for p in series.get_patches():
first = True
this_reviewers = _find_reviewers(p)
if this_reviewers:
if first:
num_reviewed += 1
first = False
reviewers = reviewers.union(this_reviewers)
series_reviewers = _find_reviewers(series)
reviewers = reviewers.union(series_reviewers)
if (num_reviewed,
num_reviewed) == series.get_num() or series_reviewers:
series.set_property("reviewed", True)
series.set_property("reviewers", list(reviewers))
if updated:
emit_event("TagsUpdate", series=series)
def on_message_added(self, event, message):
series = message.get_series_head()
if not series:
return
def newer_than(m1, m2):
return m1.version > m2.version and m1.date >= m2.date
for m in series.get_alternative_revisions():
if newer_than(m, series):
series.set_property("obsoleted-by", m.message_id)
elif newer_than(series, m):
m.set_property("obsoleted-by", series.message_id)
updated = self.update_tags(series)
for p in series.get_patches():
updated = updated or self.update_tags(p)
reviewers = set()
num_reviewed = 0
def _find_reviewers(what):
ret = set()
for rev_tag in [x for x in what.tags if x.lower().startswith(REV_BY_PREFIX.lower())]:
ret.add(parse_address(rev_tag[len(REV_BY_PREFIX):]))
return ret
for p in series.get_patches():
first = True
this_reviewers = _find_reviewers(p)
if this_reviewers:
if first:
num_reviewed += 1
first = False
reviewers = reviewers.union(this_reviewers)
series_reviewers = _find_reviewers(series)
reviewers = reviewers.union(series_reviewers)
if num_reviewed == series.get_num()[1] or series_reviewers:
series.set_property("reviewed", True)
series.set_property("reviewers", list(reviewers))
if updated:
emit_event("TagsUpdate", series=series)
def add_message_from_mbox(self, mbox, user, project_name=None):
def find_message_projects(m):
q = []
for name, addr in m.get_to() + m.get_cc():
q += Project.objects.filter(mailing_list__contains=addr)
if not q:
raise Exception("Cannot find project for message: %s" % m)
return q
m = MboxMessage(mbox)
msgid = m.get_message_id()
if project_name:
projects = [Project.object.get(name=project_name)]
else:
projects = find_message_projects(m)
for p in projects:
msg = Message(message_id=msgid,
in_reply_to=m.get_in_reply_to() or "",
date=m.get_date(),
subject=m.get_subject(),
stripped_subject=m.get_subject(strip_tags=True),
version=m.get_version(),
sender=json.dumps(m.get_from()),
receivers=json.dumps(m.get_to() + m.get_cc()),
prefixes=json.dumps(m.get_prefixes()),
is_series_head=m.is_series_head(),
is_patch=m.is_patch(),
patch_num=m.get_num()[0])
msg.project = p
if self.filter(message_id=msgid, project__name=p.name).first():
raise self.DuplicateMessageError(msgid)
msg.save_mbox(mbox)
msg.save()
emit_event("MessageAdded", message=msg)
self.update_series(msg)
return projects
def set_complete(self):
if self.is_complete:
return
self.is_complete = True
self.save()
emit_event("SeriesComplete", project=self.project, series=self)
def save(self, *args, **kwargs):
old_project = Project.objects.filter(pk=self.pk).first()
old_config = old_project.config if old_project else None
super(Project, self).save(*args, **kwargs)
if old_config != self.config:
emit_event("SetProjectConfig", obj=self)
def _update_series(self, wd, s):
logf = tempfile.NamedTemporaryFile()
project_name = s.project.name
push_to = None
try:
upstream, base_branch = self._get_project_repo_and_branch(
s.project)
cache_repo = self._poll_project(s.project)
repo = self._clone_repo(wd, cache_repo, base_branch, logf)
new_branch = "patchew/" + s.message_id
subprocess.check_call(["git", "checkout", "-q", "-b", new_branch],
cwd=repo,
stdout=logf,
stderr=logf)
base = subprocess.check_output(
["git", "log", '-n', '1', "--format=%H"],
cwd=repo).decode("utf-8").strip()
logf.write("On top of commit: %s\n" % base)
logf.flush()
for p in s.get_patches():
subprocess.check_call(
["git", "am", p.get_mbox_path()],
cwd=repo,
stdout=logf,
stderr=logf)
filter_cmd = ""
commit_message_lines = \
subprocess.check_output(["git", "log", "-n", "1",
"--format=%b"], cwd=repo)\
.decode('utf-8').splitlines()
for t in ["Message-id: %s" % p.message_id] + \
p.get_property("tags", []):
if t in commit_message_lines:
continue
filter_cmd += "echo '%s';" % t
if filter_cmd:
subprocess.check_output([
"git", "filter-branch", "-f", "--msg-filter",
"cat; " + filter_cmd, "HEAD~1.."
],
cwd=repo)
push_to = self.get_project_config(s.project, "push_to")
if push_to:
subprocess.check_call(
["git", "remote", "add", "push_remote", push_to],
cwd=repo,
stdout=logf,
stderr=logf)
# Push the new branch to remote as a new tag with the same name
subprocess.check_call(["git", "push", "-f",
"push_remote",
"refs/heads/%s:refs/tags/%s" % \
(new_branch, new_branch)],
cwd=repo,
stdout=logf, stderr=logf)
subprocess.call(["git", "push", "push_remote", base_branch],
cwd=repo,
stdout=logf,
stderr=logf)
public_repo = self.get_project_config(s.project, "public_repo")
s.set_property("git.repo", public_repo)
s.set_property("git.tag", new_branch)
s.set_property("git.base", base)
s.set_property("git.url",
self.get_project_config(s.project, "url_template").\
replace("%t", tag_name))
s.set_property("git.apply-failed", False)
emit_event("SeriesApplied", series=s)
except:
import traceback
traceback.print_exc(file=logf)
s.set_property("git.apply-failed", True)
s.set_property("git.repo", None)
s.set_property("git.tag", None)
s.set_property("git.base", None)
s.set_property("git.url", None)
finally:
logf.seek(0)
log = logf.read()
if push_to:
log = log.replace(push_to, public_repo)
s.set_property("git.apply-log", log)
def set_merged(self):
if self.is_merged:
return
self.is_merged = True
self.save()
emit_event("SeriesMerged", project=self.project, series=self)
def _drop_from_queue(self, user, m, queue):
query = QueuedSeries.objects.filter(user=user, message__in=m.get_patches() + [m],
name=queue)
for q in query:
emit_event("MessageDropping", user=user, message=q.message, queue=q)
q.delete()
def _add_to_queue(self, user, m, queue):
for x in [m] + list(m.get_patches()):
q, created = QueuedSeries.objects.get_or_create(user=user, message=x, name=queue)
if created:
emit_event("MessageQueued", user=user, message=x, queue=q)
def on_message_added(self, event, message):
series = message.get_series_head()
if not series:
return
def newer_than(m1, m2):
if m1 == m2:
return False
if m1.stripped_subject == m2.stripped_subject:
if m1.version > m2.version:
return m1.date >= m2.date
if m1.version < m2.version:
return False
if m2.is_obsolete and not m1.is_obsolete:
return True
if m1.is_obsolete and not m2.is_obsolete:
return False
return m1.date > m2.date
updated = self.update_tags(series)
for p in series.get_patches():
updated = updated or self.update_tags(p)
reviewers = set()
num_reviewed = 0
def _find_reviewers(what):
ret = set()
for rev_tag in [
x for x in what.tags
if x.lower().startswith(REV_BY_PREFIX.lower())
]:
ret.add(parse_address(rev_tag[len(REV_BY_PREFIX):]))
return ret
for p in series.get_patches():
first = True
this_reviewers = _find_reviewers(p)
if this_reviewers:
if first:
num_reviewed += 1
first = False
reviewers = reviewers.union(this_reviewers)
series_reviewers = _find_reviewers(series)
reviewers = reviewers.union(series_reviewers)
if num_reviewed == series.get_num()[1] or series_reviewers:
need_event = not series.is_reviewed
series.is_reviewed = True
series.save()
series.set_property("reviewers", list(reviewers))
if need_event:
emit_event("SeriesReviewed", series=series)
if updated:
emit_event("TagsUpdate", series=series)
if not series.topic.latest or newer_than(series, series.topic.latest):
series.topic.latest = series
series.topic.save()
for m in series.get_alternative_revisions():
if not m.is_obsolete and m.topic.latest != m:
m.is_obsolete = True
m.save()
