def _search_logs(self, request):
querystr = bytes_to_str(request.args[b"q"][0])
if b"page" in request.args:
try:
page = int(request.args[b"page"][0])
except ValueError:
page = -1
else:
page = 1
if page < 1:
log_data = "Invalid page number specified"
request.write(
str_to_bytes(
search_page_template.format(log_data=log_data,
title=self.title,
header=header,
footer=footer,
channel=self.channel)))
request.finish()
return
with self.ix.searcher() as searcher:
query = QueryParser("content", self.ix.schema).parse(querystr)
res_page = searcher.search_page(query,
page,
pagelen=self.pagelen,
sortedby="date",
reverse=True)
res_page.results.fragmenter = highlight.SentenceFragmenter(
sentencechars=u".!?\u2026", charlimit=None)
log_data = ""
for hit in res_page:
log_data += ("<ul><div><label><a href='{channel}?date="
"{date}'>{date}</a></label>".format(
channel=self.channel_link(),
date=hit["date"].strftime("%Y-%m-%d")) +
hit.highlights("content") + "</div></ul>")
else:
if not res_page.is_last_page():
log_data += "<a href='?q={}&page={}'>Next</a>".format(
querystr, page + 1)
if not res_page:
log_data = "No Logs found containg: {}".format(
htmlescape(querystr))
request.write(
str_to_bytes(
search_page_template.format(log_data=log_data,
title=self.title,
header=header,
footer=footer,
channel=self.channel_link())))
request.finish()
def _show_log(self, request):
log_data = "Log not found"
MIN_LEVEL = LEVEL_IMPORTANT
try:
MIN_LEVEL = int(request.args.get(b"level", [MIN_LEVEL])[0])
except ValueError as e:
logger.warn("Got invalid log 'level' in request arguments: "
"{level}", level=request.args[b"level"])
filename = None
date = bytes_to_str(request.args.get(b"date", [b"current"])[0])
if date == datetime.today().strftime("%Y-%m-%d"):
filename = "{}.yaml".format(self.channel)
elif date_regex.match(date):
filename = "{}.{}.yaml".format(self.channel, date)
elif date == "current":
filename = "{}.yaml".format(self.channel)
date = datetime.today().strftime("%Y-%m-%d")
if filename and os.path.isfile(os.path.join(self.log_dir, filename)):
with open(os.path.join(self.log_dir, filename)) as logfile:
log_data = '<table>'
for i, data in enumerate(yaml.load_all(logfile)):
if data["levelno"] > MIN_LEVEL:
_prepare_yaml_element(data)
log_data += line_templates[data["levelname"]].format(
index=i, **data)
log_data += '</table>'
request.write(str_to_bytes(log_page_template.format(
log_data=log_data, title=self.title, header=header,
footer=footer, channel=self.channel_link(), date=date,
Level=MIN_LEVEL)))
request.finish()
def add_resources_to_root(root):
for f in fs.listdir("resources"):
relpath = "/".join(["resources", f])
if not (f.endswith(".html") or f.endswith(".inc")):
root.putChild(
str_to_bytes(f),
File(fs.get_abs_path(relpath), defaultType="text/plain"))
def render_GET(self, request):
if b"q" not in request.args or request.args[b"q"] == ['']:
return str_to_bytes(search_page_template.format(
log_data="", title=self.title, header=header,
footer=footer, channel=self.channel_link()))
d = deferLater(reactor, 0, self._search_logs, request)
d.addErrback(_onError, request)
return NOT_DONE_YET
def render_GET(self, request):
data = ""
for channel in self.channels:
data += "<a href='{0}'>{0}</a><br/>".format(channel.lstrip("#"))
return str_to_bytes(base_page_template.format(title=self.title,
data=data,
header=header,
footer=footer))
def render_GET(self, request):
data = ""
for channel in self.channels:
data += "<a href='{0}'>{0}</a><br/>".format(channel.lstrip("#"))
return str_to_bytes(
base_page_template.format(title=self.title,
data=data,
header=header,
footer=footer))
def _search_logs(self, request):
querystr = unicode(request.args[b"q"][0], "utf-8")
if b"page" in request.args:
try:
page = int(request.args[b"page"][0])
except ValueError:
page = -1
else:
page = 1
if page < 1:
log_data = "Invalid page number specified"
request.write(str_to_bytes(search_page_template.format(
log_data=log_data, title=self.title, header=header,
footer=footer, channel=self.channel)))
request.finish()
return
with self.ix.searcher() as searcher:
query = QueryParser("content", self.ix.schema).parse(querystr)
res_page = searcher.search_page(query, page,
pagelen=self.pagelen,
sortedby="date", reverse=True)
res_page.results.fragmenter = highlight.SentenceFragmenter(
sentencechars=u".!?\u2026", charlimit=None)
log_data = ""
for hit in res_page:
log_data += ("<ul><div><label><a href='{channel}?date="
"{date}'>{date}</a></label>".format(
channel=self.channel_link(),
date=hit["date"].strftime("%Y-%m-%d")) +
hit.highlights("content") +
"</div></ul>")
else:
if not res_page.is_last_page():
log_data += "<a href='?q={}&page={}'>Next</a>".format(
querystr, page + 1)
if not res_page:
log_data = "No Logs found containg: {}".format(
htmlescape(querystr))
if sys.version_info.major < 3:
log_data = log_data.encode("utf-8")
request.write(str_to_bytes(search_page_template.format(
log_data=log_data, title=self.title, header=header,
footer=footer, channel=self.channel_link())))
request.finish()
def render_GET(self, request):
if b"q" not in request.args or request.args[b"q"] == ['']:
return str_to_bytes(
search_page_template.format(log_data="",
title=self.title,
header=header,
footer=footer,
channel=self.channel_link()))
if self.ix is None:
return str_to_bytes(
search_page_template.format(
log_data="Indexing..., Please try again later",
title=self.title,
header=header,
footer=footer,
channel=self.channel_link()))
d = deferLater(reactor, 0, self._search_logs, request)
d.addErrback(_onError, request)
return NOT_DONE_YET
def render_POST(self, request):
body = request.content.read()
data = json.loads(bytes_to_str(body))
service = None
# GitHub
if request.getHeader(b"X-GitHub-Event"):
eventtype = request.getHeader(b"X-GitHub-Event")
sig = request.getHeader(b"X-Hub-Signature")
if sig:
sig = sig[5:]
service = "github"
# Gitlab
elif request.getHeader(b"X-Gitlab-Event"):
eventtype = data["object_kind"]
sig = request.getHeader(b"X-Gitlab-Token")
service = "gitlab"
# other: not implemented
else:
request.setResponseCode(403)
return b""
eventtype = bytes_to_str(eventtype)
secret = None
if service == "github":
secret = self.github_secret
elif service == "gitlab":
secret = self.gitlab_secret
if secret:
secret = str_to_bytes(secret)
h = hmac.new(secret, body, sha1)
if codecs.encode(h.digest(), "hex") != sig:
self.log.warn("Request's signature does not correspond"
" with the given secret - ignoring request")
request.setResponseCode(200)
return b""
# filtering: inject eventtype for filtering
data["eventtype"] = eventtype
if self.filter_event(data):
self.log.debug("filtering out event {event}", event=data)
elif hasattr(self, "on_{}_{}".format(service, eventtype)):
reactor.callLater(
0, getattr(self, "on_{}_{}".format(service, eventtype)), data)
else:
self.log.warn(
"Event {eventtype} not implemented for service "
"{service}",
eventtype=eventtype,
service=service)
# always return 200
request.setResponseCode(200)
return b""
def __init__(self, config):
super(BasePage, self).__init__()
self.title = config["HTTPLogServer"].get("title", "PyTIBot Log Server")
# add channel logs
self.channels = config["HTTPLogServer"].get("channels", [])
search_pagelen = config["HTTPLogServer"].get("search_pagelen", 5)
for channel in self.channels:
name = channel.lstrip("#")
self.putChild(str_to_bytes(name),
LogPage(name, log.get_channellog_dir(config),
"#{} - {}".format(name, self.title),
search_pagelen))
# add resources
add_resources_to_root(self)
def __init__(self, config):
super(BasePage, self).__init__()
self.title = config["HTTPLogServer"].get("title", "PyTIBot Log Server")
# add channel logs
self.channels = config["HTTPLogServer"].get("channels", [])
search_pagelen = config["HTTPLogServer"].get("search_pagelen", 5)
indexer_procs = config["HTTPLogServer"].get("indexer_procs", 1)
for channel in self.channels:
name = channel.lstrip("#")
self.putChild(
str_to_bytes(name),
LogPage(name, log.get_channellog_dir(config),
"#{} - {}".format(name, self.title), search_pagelen,
indexer_procs))
# add resources
add_resources_to_root(self)
def render_POST(self, request):
body = request.content.read()
data = json.loads(bytes_to_str(body))
service = None
# GitHub
if request.getHeader(b"X-GitHub-Event"):
eventtype = request.getHeader(b"X-GitHub-Event")
sig = request.getHeader(b"X-Hub-Signature")
if sig:
sig = sig[5:]
service = "github"
# Gitlab
elif request.getHeader(b"X-Gitlab-Event"):
eventtype = data["object_kind"]
sig = request.getHeader(b"X-Gitlab-Token")
service = "gitlab"
eventtype = bytes_to_str(eventtype)
secret = None
if service == "github":
secret = self.github_secret
elif service == "gitlab":
secret = self.gitlab_secret
if secret:
secret = str_to_bytes(secret)
h = hmac.new(secret, body, sha1)
if codecs.encode(h.digest(), "hex") != sig:
self.log.warn("Request's signature does not correspond"
" with the given secret - ignoring request")
request.setResponseCode(200)
return b""
if hasattr(self, "on_{}_{}".format(service, eventtype)):
reactor.callLater(0, getattr(self, "on_{}_{}".format(service,
eventtype)),
data)
else:
self.log.warn("Event {eventtype} not implemented for service "
"{service}", eventtype=eventtype, service=service)
# always return 200
request.setResponseCode(200)
return b""
def _show_log(self, request):
log_data = "Log not found"
MIN_LEVEL = LEVEL_IMPORTANT
try:
MIN_LEVEL = int(request.args.get(b"level", [MIN_LEVEL])[0])
except ValueError as e:
logger.warn(
"Got invalid log 'level' in request arguments: "
"{level}",
level=request.args[b"level"])
filename = None
date = bytes_to_str(request.args.get(b"date", [b"current"])[0])
if date == datetime.today().strftime("%Y-%m-%d"):
filename = "{}.yaml".format(self.channel)
elif date_regex.match(date):
filename = "{}.{}.yaml".format(self.channel, date)
elif date == "current":
filename = "{}.yaml".format(self.channel)
date = datetime.today().strftime("%Y-%m-%d")
if filename and os.path.isfile(os.path.join(self.log_dir, filename)):
with open(os.path.join(self.log_dir, filename)) as logfile:
log_data = '<table>'
for i, data in enumerate(yaml.full_load_all(logfile)):
if data["levelno"] > MIN_LEVEL:
_prepare_yaml_element(data)
log_data += line_templates[data["levelname"]].format(
index=i, **data)
log_data += '</table>'
request.write(
str_to_bytes(
log_page_template.format(log_data=log_data,
title=self.title,
header=header,
footer=footer,
channel=self.channel_link(),
date=date,
Level=MIN_LEVEL)))
request.finish()
def add_resources_to_root(root):
for f in fs.listdir("resources"):
relpath = "/".join(["resources", f])
if not (f.endswith(".html") or f.endswith(".inc")):
root.putChild(str_to_bytes(f), File(fs.get_abs_path(relpath),
defaultType="text/plain"))
def sendLine(self, line):
# Python3 compatibility: ensure that line is a bytes string
super(STDIOInterface, self).sendLine(str_to_bytes(line))