class TestingModule(PatchewModule):
"""Testing module"""
name = "testing"
allowed_groups = ("testers", )
result_data_serializer_class = ResultDataSerializer
test_schema = schema.ArraySchema(
"{name}",
"Test",
desc="Test spec",
members=[
schema.BooleanSchema("enabled",
"Enabled",
desc="Whether this test is enabled",
default=True),
schema.StringSchema("requirements",
"Requirements",
desc="List of requirements of the test"),
schema.IntegerSchema("timeout",
"Timeout",
default=3600,
desc="Timeout for the test"),
schema.StringSchema(
"script",
"Test script",
desc="The testing script",
default=TESTING_SCRIPT_DEFAULT,
multiline=True,
required=True,
),
],
)
requirement_schema = schema.ArraySchema(
"{name}",
"Requirement",
desc="Test requirement spec",
members=[
schema.StringSchema(
"script",
"Probe script",
desc="The probing script for this requirement",
default="#!/bin/bash\ntrue",
multiline=True,
required=True,
)
],
)
project_config_schema = schema.ArraySchema(
"testing",
desc="Configuration for testing module",
members=[
schema.MapSchema("tests",
"Tests",
desc="Testing specs",
item=test_schema),
schema.MapSchema(
"requirements",
"Requirements",
desc="Requirement specs",
item=requirement_schema,
),
],
)
def __init__(self):
global _instance
assert _instance == None
_instance = self
declare_event(
"TestingReport",
user="******",
tester="the name of the tester",
obj="the object (series or project) which the test is for",
passed="True if the test is passed",
test="test name",
log="test log",
log_url="URL to test log (text)",
html_log_url="URL to test log (HTML)",
is_timeout="whether the test has timeout",
)
register_handler("SetProperty", self.on_set_property)
register_handler("SetProjectConfig", self.on_set_config)
register_handler("ResultUpdate", self.on_result_update)
def on_set_property(self, evt, obj, name, value, old_value):
if isinstance(obj,
Project) and name == "git.head" and old_value != value:
self.clear_and_start_testing(obj)
def on_set_config(self, evt, obj):
self.project_recalc_pending_tests(obj)
def get_msg_base_tags(self, msg):
return [t for t in msg.tags if t.lower().startswith("based-on:")]
def on_result_update(self, evt, obj, old_status, result):
if result.name.startswith("testing.") and result.status != old_status:
if "tester" in result.data:
po = obj if isinstance(obj, Project) else obj.project
_instance.tester_check_in(po, result.data["tester"])
if not self.get_testing_results(
obj, status__in=(Result.PENDING, Result.RUNNING)).exists():
obj.set_property("testing.tested-head", result.data["head"])
if isinstance(obj, Message):
obj.is_tested = True
obj.save()
obj.set_property("testing.tested-base",
self.get_msg_base_tags(obj))
if isinstance(obj, Project):
# cache the last result so that badges are not affected by RUNNING state
failures = obj.get_property("testing.failures", [])
if result.status == result.SUCCESS and result.name in failures:
failures.remove(result.name)
obj.set_property("testing.failures", list(failures))
if result.status == result.FAILURE and result.name not in failures:
failures.append(result.name)
obj.set_property("testing.failures", list(failures))
if result.name != "git":
return
if (isinstance(obj, Message) and obj.is_series_head
and result.status == result.SUCCESS and result.data.get("tag")
and result.data.get("repo")):
tested_base = obj.get_property("testing.tested-base")
if tested_base is None or tested_base != self.get_msg_base_tags(
obj):
self.clear_and_start_testing(obj)
def filter_testing_results(self, queryset, *args, **kwargs):
return queryset.filter(name__startswith="testing.", *args, **kwargs)
def get_testing_results(self, obj, *args, **kwargs):
return self.filter_testing_results(obj.results, *args, **kwargs)
def get_testing_result(self, obj, name):
try:
return obj.results.get(name="testing." + name)
except:
raise Http404("Test doesn't exist")
def get_test_name(self, result):
return result.name[len("testing."):]
def recalc_pending_tests(self, obj):
test_dict = self.get_tests(obj)
all_tests = set(
(k for k, v in test_dict.items() if v.get("enabled", False)))
for r in self.get_testing_results(obj, status=Result.PENDING):
r.delete()
if len(all_tests):
done_tests = [
self.get_test_name(r) for r in self.get_testing_results(obj)
]
for tn in all_tests:
if not tn in done_tests:
obj.create_result(name="testing." + tn,
status=Result.PENDING).save()
if isinstance(obj, Message):
is_tested = len(all_tests) and len(done_tests) == len(all_tests)
if is_tested != obj.is_tested:
obj.is_tested = is_tested
obj.save()
def project_recalc_pending_tests(self, project):
self.recalc_pending_tests(project)
# Only operate on messages for which testing has not completed yet.
message_ids = self.filter_testing_results(
MessageResult.objects,
message__project=project,
status=Result.PENDING).values("message_id")
messages = Message.objects.filter(id__in=message_ids)
for obj in messages:
self.recalc_pending_tests(obj)
def clear_and_start_testing(self, obj, test=""):
obj.set_property("testing.tested-head", None)
if isinstance(obj, Message):
obj.is_tested = False
obj.save()
if test:
r = self.get_testing_result(obj, test)
if r:
r.delete()
else:
for r in self.get_testing_results(obj):
r.delete()
self.recalc_pending_tests(obj)
def www_view_testing_reset(self, request, project_or_series):
if not request.user.is_authenticated:
return HttpResponseForbidden()
if request.GET.get("type") == "project":
obj = Project.objects.filter(name=project_or_series).first()
if not obj.maintained_by(request.user):
raise PermissionDenied()
else:
obj = Message.objects.find_series(project_or_series)
if not obj:
raise Http404("Not found: " + project_or_series)
self.clear_and_start_testing(obj, request.GET.get("test", ""))
return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
def www_view_badge(self, request, project, ext):
po = Project.objects.filter(name=project).first()
if po.get_property("testing.failures"):
return HttpResponseRedirect(
"https://img.shields.io/badge/patchew-failing-critical." + ext)
else:
return HttpResponseRedirect(
"https://img.shields.io/badge/patchew-passing-success." + ext)
def www_url_hook(self, urlpatterns):
urlpatterns.append(
url(
r"^testing-reset/(?P<project_or_series>.*)/",
self.www_view_testing_reset,
name="testing-reset",
))
urlpatterns.append(
url(
r"^logs/(?P<project_or_series>.*)/testing.(?P<testing_name>.*)/",
TestingLogViewer.as_view(),
name="testing-log",
))
urlpatterns.append(
url(
r"^(?P<project>[^/]*)/badge.(?P<ext>svg|png)$",
self.www_view_badge,
name="testing-badge",
))
def api_url_hook(self, urlpatterns):
urlpatterns.append(
url(
r"^v1/projects/(?P<pk>[^/]*)/get-test/$",
GetTestView.as_view(),
name="get-test",
))
def reverse_testing_log(self, obj, test, request=None, html=False):
if isinstance(obj, Message):
log_url = (reverse(
"testing-log",
kwargs={
"project_or_series": obj.message_id,
"testing_name": test
},
) + "?type=message")
else:
assert isinstance(obj, Project)
log_url = (reverse(
"testing-log",
kwargs={
"project_or_series": obj.name,
"testing_name": test
},
) + "?type=project")
if html:
log_url += "&html=1"
# Generate a full URL, including the host and port, for use in
# email notifications
if request:
log_url = request.build_absolute_uri(log_url)
return log_url
def add_test_report(
self,
request,
project,
tester,
test,
head,
base,
identity,
passed,
log,
is_timeout,
):
# Find a project or series depending on the test type and assign it to obj
if identity["type"] == "project":
obj = Project.objects.get(name=project)
project = obj.name
elif identity["type"] == "series":
message_id = identity["message-id"]
obj = Message.objects.find_series(message_id, project)
if not obj:
raise Exception("Series doesn't exist")
project = obj.project.name
user = request.user
r = self.get_testing_result(obj, test)
r.data = {
"is_timeout": is_timeout,
"user": user.username,
"head": head,
"tester": tester or user.username,
}
r.log = log
r.status = Result.SUCCESS if passed else Result.FAILURE
r.save()
log_url = self.reverse_testing_log(obj, test, request=request)
html_log_url = self.reverse_testing_log(obj,
test,
request=request,
html=True)
emit_event(
"TestingReport",
tester=tester,
user=user.username,
obj=obj,
passed=passed,
test=test,
log=log,
log_url=log_url,
html_log_url=html_log_url,
is_timeout=is_timeout,
)
def get_tests(self, obj):
if isinstance(obj, Message):
obj = obj.project
return self.get_project_config(obj).get("tests", {})
def _build_reset_ops(self, obj):
if isinstance(obj, Message):
typearg = "type=message"
url = reverse("testing-reset",
kwargs={"project_or_series": obj.message_id})
else:
assert isinstance(obj, Project)
url = reverse("testing-reset",
kwargs={"project_or_series": obj.name})
typearg = "type=project"
url += "?" + typearg
ret = [{
"url": url,
"title": "Reset all testing states",
"class": "warning",
"icon": "refresh",
}]
for r in self.get_testing_results(obj, ~Q(status=Result.PENDING)):
tn = self.get_test_name(r)
ret.append({
"url":
url + "&test=" + tn,
"title":
format_html("Reset <b>{}</b> testing state", tn),
"class":
"warning",
"icon":
"refresh",
})
return ret
def prepare_message_hook(self, request, message, detailed):
if not message.is_series_head:
return
if (message.project.maintained_by(request.user)
and self.get_testing_results(
message, ~Q(status=Result.PENDING)).exists()):
message.extra_ops += self._build_reset_ops(message)
if self.get_testing_results(message, status=Result.FAILURE).exists():
message.status_tags.append({
"title":
"Testing failed",
"url":
reverse(
"series_detail",
kwargs={
"project": message.project.name,
"message_id": message.message_id,
},
),
"type":
"danger",
"char":
"T",
})
elif message.is_tested:
message.status_tags.append({
"title":
"Testing passed",
"url":
reverse(
"series_detail",
kwargs={
"project": message.project.name,
"message_id": message.message_id,
},
),
"type":
"success",
"char":
"T",
})
def get_result_log_url(self, result):
tn = result.name[len("testing."):]
return self.reverse_testing_log(result.obj, tn, html=False)
def render_result(self, result):
if not result.is_completed():
return None
pn = result.name
tn = pn[len("testing."):]
log_url = result.get_log_url()
html_log_url = log_url + "&html=1"
passed_str = "failed" if result.is_failure() else "passed"
return format_html(
'Test <b>{}</b> <a class="cbox-log" data-link="{}" href="{}">{}</a>',
tn,
html_log_url,
log_url,
passed_str,
)
def check_active_testers(self, project):
at = []
for tn, v in project.get_property("testing.check_in", {}).items():
age = time.time() - v
if age > 10 * 60:
continue
at.append("%s (%dmin)" % (tn, math.ceil(age / 60)))
if not at:
return
project.extra_status.append({
"icon": "fa-refresh fa-spin",
"html": "Active testers: " + ", ".join(at)
})
def prepare_project_hook(self, request, project):
if not project.maintained_by(request.user):
return
project.extra_info.append({
"title":
"Testing configuration",
"class":
"info",
"content_html":
self.build_config_html(request, project),
})
self.check_active_testers(project)
if (project.maintained_by(request.user) and self.get_testing_results(
project, ~Q(status=Result.PENDING)).exists()):
project.extra_ops += self._build_reset_ops(project)
def get_capability_probes(self, project):
props = self.get_project_config(project).get("requirements", {})
return {k: v["script"] for k, v in props.items()}
def get_testing_probes(self, project, request, format):
return self.get_capability_probes(project)
def rest_project_fields_hook(self, request, fields):
fields["testing_probes"] = PluginMethodField(obj=self)
def tester_check_in(self, project, tester):
assert project
assert tester
po = Project.objects.filter(name=project).first()
if not po:
return
po.set_property("testing.check_in." + tester, time.time())
class EmailModule(PatchewModule):
"""
Documentation
-------------
Email information is configured in "INI" style:
""" + _default_config
name = "email" # The notify method name
default_config = _default_config
email_schema = schema.ArraySchema(
"{name}",
"Email Notification",
desc="Email notification",
members=[
schema.EnumSchema(
"event",
"Event",
enums=lambda: get_events_info(),
required=True,
desc="Which event to trigger the email notification",
),
schema.BooleanSchema(
"enabled", "Enabled", desc="Whether this event is enabled", default=True
),
schema.BooleanSchema(
"reply_to_all",
"Reply to all",
desc='If set, Cc all the receipients of the email message associated to the event. Also, if set the original sender of the email message will be a recipient even if the "to" field is nonempty',
default=False,
),
schema.BooleanSchema(
"in_reply_to",
"Set In-Reply-To",
desc="Whether to set In-Reply-To to the message id, if the event has an associated email message",
default=True,
),
schema.BooleanSchema(
"set_reply_to",
"Set Reply-To",
desc="Whether to set Reply-To to the project mailing list, if the event has an associated email message",
default=True,
),
schema.BooleanSchema(
"reply_subject",
"Set replying subject",
desc='Whether to set Subject to "Re: xxx", if the event has an associated email message',
default=True,
),
schema.BooleanSchema(
"to_user",
"Send to user",
desc="Whether to set To to a user email, if the event has an associated user",
default=False,
),
schema.StringSchema("to", "To", desc="Send email to"),
schema.StringSchema("cc", "Cc", desc="Cc list"),
schema.StringSchema(
"subject_template",
"Subject template",
desc="""The django template for subject""",
required=True,
),
schema.StringSchema(
"body_template",
"Body template",
desc="The django template for email body.",
multiline=True,
required=True,
),
],
)
project_config_schema = schema.ArraySchema(
"email",
desc="Configuration for email module",
members=[
schema.MapSchema(
"notifications",
"Email notifications",
desc="Email notifications",
item=email_schema,
)
],
)
def __init__(self):
register_handler(None, self.on_event)
def _get_smtp(self):
server = self.get_config("smtp", "server")
port = self.get_config("smtp", "port")
username = self.get_config("smtp", "username")
password = self.get_config("smtp", "password")
ssl = self.get_config("smtp", "ssl", "getboolean")
if settings.DEBUG:
return DebugSMTP()
elif ssl:
smtp = smtplib.SMTP_SSL(server, port)
else:
smtp = smtplib.SMTP(server, port)
if self.get_config("smtp", "auth", "getboolean"):
smtp.login(username, password)
return smtp
def _send_series_recurse(self, sendmethod, s):
sendmethod(s)
for i in s.get_replies():
self._send_series_recurse(sendmethod, i)
def _smtp_send(self, to, cc, message):
from_addr = self.get_config("smtp", "from")
message["Resent-From"] = message["From"]
for k, v in [("From", from_addr), ("To", to), ("Cc", cc)]:
if not v:
continue
if isinstance(v, list):
v = ", ".join(v)
try:
message.replace_header(k, v)
except KeyError:
message[k] = v
smtp = self._get_smtp()
recipients = []
for x in [to, cc]:
if not x:
continue
if isinstance(x, str):
recipients += [x]
elif isinstance(x, list):
recipients += x
smtp.sendmail(from_addr, recipients, message.as_string())
def www_view_email_bounce(self, request, message_id):
if not request.user.is_authenticated:
raise PermissionDenied()
m = Message.objects.find_series(message_id)
if not m:
raise Http404("Series not found: " + message_id)
def send_one(m):
msg = m.get_mbox()
message = email.message_from_string(msg)
self._smtp_send(request.user.email, None, message)
self._send_series_recurse(send_one, m)
return HttpResponse("email bounced")
def www_url_hook(self, urlpatterns):
urlpatterns.append(
url(
r"^email-bounce/(?P<message_id>.*)/",
self.www_view_email_bounce,
name="email-bounce",
)
)
def prepare_message_hook(self, request, message, detailed):
if not detailed:
return
if message.is_series_head and request.user.is_authenticated and request.user.email:
message.extra_ops.append(
{
"url": reverse(
"email-bounce", kwargs={"message_id": message.message_id}
),
"icon": "mail-forward",
"title": "Bounce to me",
}
)
def _sections_by_event(self, event):
conf = self.get_config_obj()
for sec in conf.sections():
if sec.startswith("mail ") and conf.get(sec, "event") == event:
yield sec
def _send_email(self, to, cc, headers, body):
message = email.message.Message()
for k, v in headers.items():
message[k] = v
message.set_payload(body, charset="utf-8")
self._smtp_send(to, cc, message)
def gen_message_id(self):
return "<*****@*****.**>" % uuid.uuid1()
def get_notifications(self, project):
return self.get_project_config(project).get("notifications", {})
def on_event(self, event, **params):
class EmailCancelled(Exception):
pass
po = None
mo = None
for v in list(params.values()):
if isinstance(v, Message):
mo = v
po = mo.project
break
elif isinstance(v, Project):
po = v
break
if not po:
return
for nt in list(self.get_notifications(po).values()):
headers = {}
if not nt["enabled"]:
continue
if nt["event"] != event:
continue
def cancel_email():
raise EmailCancelled
params["cancel"] = cancel_email
ctx = Context(params, autoescape=False)
try:
subject = Template(nt["subject_template"]).render(ctx).strip()
body = Template(nt["body_template"]).render(ctx).strip()
to = [x.strip() for x in Template(nt["to"]).render(ctx).strip().split()]
cc = [x.strip() for x in Template(nt["cc"]).render(ctx).strip().split()]
except EmailCancelled:
continue
if mo:
if nt["reply_to_all"] or not len(to):
to += [mo.get_sender_addr()]
if nt["reply_to_all"]:
cc += [x[1] for x in mo.recipients]
if mo and nt["in_reply_to"]:
headers["In-Reply-To"] = "<%s>" % mo.message_id
if mo and nt["set_reply_to"]:
headers["Reply-To"] = "<%s>" % mo.project.mailing_list
if nt["reply_subject"] and mo:
subject = (
"Re: " + mo.subject
if not mo.subject.startswith("Re:")
else mo.subject
)
if nt["to_user"] and "user" in params and params["user"].email:
to += params["user"].email
if not (subject and body and (to or cc)):
continue
headers["Subject"] = subject
headers["Message-ID"] = email.utils.make_msgid()
self._send_email(to, cc, headers, body)
def prepare_project_hook(self, request, project):
if not project.maintained_by(request.user):
return
project.extra_info.append(
{
"title": "Email notifications",
"class": "info",
"content_html": self.build_config_html(request, project),
}
)
class GitModule(PatchewModule):
"""Git module"""
name = "git"
allowed_groups = ("importers", )
result_data_serializer_class = ResultDataSerializer
project_config_schema = schema.ArraySchema(
"git",
desc="Configuration for git module",
members=[
schema.StringSchema("push_to",
"Push remote",
desc="Remote to push to",
required=True),
schema.StringSchema("public_repo",
"Public repo",
desc="Publicly visible repo URL"),
schema.BooleanSchema(
"use_git_push_option",
"Enable git push options",
desc="Whether the push remote accepts git push options"),
schema.StringSchema(
"url_template",
"URL template",
desc=
"Publicly visible URL template for applied branch, where %t will be replaced by the applied tag name",
required=True,
),
],
)
def __init__(self):
global _instance
assert _instance == None
_instance = self
# Make sure git is available
subprocess.check_output(["git", "version"])
declare_event("ProjectGitUpdate", project="the updated project name")
declare_event("SeriesApplied", series="the object of applied series")
register_handler("SeriesComplete", self.on_series_update)
register_handler("TagsUpdate", self.on_tags_update)
def mark_as_pending_apply(self, series, data={}):
r = series.git_result or series.create_result(name="git")
r.log = None
r.status = Result.PENDING
r.data = data
r.save()
def on_tags_update(self, event, series, **params):
if series.is_complete:
self.mark_as_pending_apply(series, {
'git.push_options': 'ci.skip',
})
def on_series_update(self, event, series, **params):
if series.is_complete:
self.mark_as_pending_apply(series)
def _is_repo(self, path):
if not os.path.isdir(path):
return False
if 0 != subprocess.call(
["git", "rev-parse", "--is-bare-repository"],
cwd=path,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
):
return False
return True
def get_based_on(self, message, request, format):
git_base = self.get_base(message)
return git_base.data if git_base else None
def get_mirror(self, po, request, format):
response = {}
config = self.get_project_config(po)
if "push_to" in config:
response["pushurl"] = config["push_to"]
if "public_repo" in config:
response["url"] = config["public_repo"]
head = po.get_property("git.head")
if head:
response["head"] = head
return response
def rest_project_fields_hook(self, request, fields):
fields["mirror"] = PluginMethodField(obj=self, required=False)
def rest_series_fields_hook(self, request, fields, detailed):
fields["based_on"] = PluginMethodField(obj=self, required=False)
def get_projects_prepare_hook(self, project, response):
response["git.head"] = project.get_property("git.head")
config = self.get_project_config(project)
if "push_to" in config:
response["git.push_to"] = config["push_to"]
def prepare_message_hook(self, request, message, detailed):
if not message.is_series_head:
return
r = message.git_result
if r and r.is_completed():
if r.is_failure():
title = "Failed in applying to current master"
message.status_tags.append({
"title": title,
"type": "default",
"char": "G"
})
else:
git_url = r.data.get("url")
if git_url:
git_repo = r.data["repo"]
git_tag = r.data["tag"]
message.status_tags.append({
"url":
git_url,
"title":
format_html("Applied as tag {} in repo {}", git_tag,
git_repo),
"type":
"info",
"char":
"G",
})
else:
message.status_tags.append({
"title":
format_html("Patches applied successfully"),
"type":
"info",
"char":
"G",
})
if request.user.is_authenticated:
url = reverse("git_reset",
kwargs={"series": message.message_id})
message.extra_ops.append({
"url": url,
"icon": "refresh",
"title": "Git reset",
"class": "warning",
})
def render_result(self, result):
if not result.is_completed():
return None
log_url = result.get_log_url()
html_log_url = log_url + "?html=1"
colorbox_a = format_html(
'<a class="cbox-log" data-link="{}" href="{}">apply log</a>',
html_log_url,
log_url,
)
if result.is_failure():
return format_html("Failed in applying to current master ({})",
colorbox_a)
else:
if "url" in result.data:
s = format_html('<a href="{}">tree</a>, {}',
result.data["url"], colorbox_a)
else:
s = colorbox_a
s = format_html("Patches applied successfully ({})", s)
if "repo" in result.data and "tag" in result.data:
git_repo = result.data["repo"]
git_tag = result.data["tag"]
if git_tag.startswith("refs/tags/"):
git_tag = git_tag[5:]
s += format_html("<br/><samp>git fetch {} {}</samp>", git_repo,
git_tag)
return s
def get_result_log_url(self, result):
return reverse("git-log", kwargs={"series": result.obj.message_id})
def prepare_project_hook(self, request, project):
if not project.maintained_by(request.user):
return
project.extra_info.append({
"title":
"Git configuration",
"class":
"info",
"content_html":
self.build_config_html(request, project),
})
def get_base(self, series):
for tag in series.tags:
if not tag.startswith("Based-on:"):
continue
base = Message.objects.find_series_from_tag(tag, series.project)
if not base:
return None
r = base.git_result
return r if r and r.data.get("repo") else None
def www_view_git_reset(self, request, series):
if not request.user.is_authenticated:
raise PermissionDenied
obj = Message.objects.find_series(series)
if not obj:
raise Http404("Not found: " + series)
self.mark_as_pending_apply(obj)
return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
def www_url_hook(self, urlpatterns):
urlpatterns.append(
url(r"^git-reset/(?P<series>.*)/",
self.www_view_git_reset,
name="git_reset"))
urlpatterns.append(
url(r"^logs/(?P<series>.*)/git/",
GitLogViewer.as_view(),
name="git-log"))
def api_url_hook(self, urlpatterns):
urlpatterns.append(
url(
r"^v1/series/unapplied/$",
UnappliedSeriesView.as_view(),
name="unapplied",
))
def pending_series(self, target_repo):
q = Message.objects.filter(results__name="git",
results__status="pending")
if target_repo is not None and target_repo != "":
# Postgres could use JSON fields instead. Fortunately projects are
# few so this is cheap
def match_target_repo(config, target_repo):
push_to = config.get("git", {}).get("push_to")
if push_to is None:
return False
if target_repo[-1] != "/":
return push_to == target_repo or push_to.startswith(
target_repo + "/")
else:
return push_to.startswith(target_repo)
projects = Project.objects.values_list("id", "config").all()
projects = [
pid for pid, config in projects
if match_target_repo(config, target_repo)
]
q = q.filter(project__pk__in=projects)
return q