def scanReports():
global fakeSubscription, interval, subscriptions, startTime
result = []
for dbreport in getReports(startTime):
matchSubscriptions = {}
recipients = []
reportSubscriptions = getReportSubscriptions(dbreport['guid'])
if dbreport['type'] == 'false positive' or dbreport[
'type'] == 'false negative':
for subscription in reportSubscriptions:
subscriptionID = subscription.get('url', 'unknown')
if subscriptionID in subscriptions:
if subscription.get('hasmatches', 0) > 0:
matchSubscriptions[subscriptionID] = subscriptions[
subscriptionID]
# Send false negatives to all subscription authors, false positives
# only to subscriptions with matching filters
if dbreport['type'] == 'false negative' or subscription.get(
'hasmatches', 0) > 0:
recipients.append(subscriptions[subscriptionID])
elif interval != 'week':
# Send type "other" to fake subscription - daily reports
recipients.append(fakeSubscription)
subscriptionID = fakeSubscription.get('url', 'unknown')
if len(recipients) == 0:
continue
report = {
'url':
get_config().get('reports', 'urlRoot') + dbreport['guid'] +
'#secret=' + calculateReportSecret(dbreport['guid']),
'weight':
calculateReportWeight(dbreport, reportSubscriptions),
'site':
dbreport['site'],
'subscriptions':
recipients,
'comment':
re.sub(r'[\x00-\x20]', r' ', dbreport['comment'])
if dbreport['comment'] != None else '',
'type':
dbreport['type'],
'numSubscriptions':
len(reportSubscriptions),
'matchSubscriptions':
matchSubscriptions.values(),
'contact':
dbreport['contact'],
'hasscreenshot':
dbreport['hasscreenshot'],
'knownIssues':
dbreport['knownissues'],
}
result.append(report)
return result
def handleRequest(environ, start_response):
setupStderr(environ['wsgi.errors'])
if environ['REQUEST_METHOD'].upper() != 'POST' or not environ.get('CONTENT_TYPE', '').startswith('application/x-www-form-urlencoded'):
return showError('Unsupported request method', start_response)
try:
request_body_length = int(environ['CONTENT_LENGTH'])
except:
return showError('Invalid or missing Content-Length header', start_response)
request_body = environ['wsgi.input'].read(request_body_length)
params = {}
for key, value in parse_qsl(request_body):
params[key] = value.decode('utf-8')
guid = params.get('guid', '').lower()
if not re.match(r'^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$', guid):
return showError('Invalid or missing report GUID', start_response)
reportData = getReport(guid)
if reportData == None:
return showError('Report does not exist', start_response)
secret = calculateReportSecret(guid)
if params.get('secret', '') != secret and params.get('secret', '') != calculateReportSecret_compat(guid):
return showError('Wrong secret value', start_response)
reportData['status'] = params.get('status', '')
if len(reportData['status']) > 1024:
reportData['status'] = reportData['status'][:1024]
oldusefulness = reportData.get('usefulness', '0')
reportData['usefulness'] = params.get('usefulness', '0')
if ('email' in reportData):
updateUserUsefulness(getUserId(reportData['email']), reportData['usefulness'], oldusefulness)
saveReport(guid, reportData)
if params.get('notify', '') and 'email' in reportData:
email = reportData['email']
email = re.sub(r' at ', r'@', email)
email = re.sub(r' dot ', r'.', email)
if re.match(r'^[\w.%+-]+@[\w.%+-]+(\.[\w.%+-]+)+', email):
sendUpdateNotification({
'email': email,
'url': get_config().get('reports', 'urlRoot') + guid,
'status': reportData['status'],
})
newURL = get_config().get('reports', 'urlRoot') + guid
newURL += '?updated=' + str(int(random.uniform(0, 10000)))
newURL += '#secret=' + secret
start_response('302 Found', [('Location', newURL.encode('utf-8'))])
return []
def handleRequest(environ, start_response):
setupStderr(environ['wsgi.errors'])
if environ['REQUEST_METHOD'].upper() != 'POST' or not environ.get('CONTENT_TYPE', '').startswith('application/x-www-form-urlencoded'):
return showError('Unsupported request method', start_response)
try:
request_body_length = int(environ['CONTENT_LENGTH'])
except:
return showError('Invalid or missing Content-Length header', start_response)
request_body = environ['wsgi.input'].read(request_body_length)
params = {}
for key, value in parse_qsl(request_body):
params[key] = value.decode('utf-8')
guid = params.get('guid', '').lower()
if not re.match(r'^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$', guid):
return showError('Invalid or missing report GUID', start_response)
reportData = getReport(guid)
if reportData == None:
return showError('Report does not exist', start_response)
secret = calculateReportSecret(guid)
if params.get('secret', '') != secret and params.get('secret', '') != calculateReportSecret_compat(guid):
return showError('Wrong secret value', start_response)
reportData['status'] = params.get('status', '')
if len(reportData['status']) > 1024:
reportData['status'] = reportData['status'][:1024]
oldusefulness = reportData.get('usefulness', '0')
reportData['usefulness'] = params.get('usefulness', '0')
if 'email' in reportData:
updateUserUsefulness(getUserId(reportData['email']), reportData['usefulness'], oldusefulness)
saveReport(guid, reportData)
if params.get('notify', '') and 'email' in reportData:
email = reportData['email']
email = re.sub(r' at ', r'@', email)
email = re.sub(r' dot ', r'.', email)
if re.match(r'^[\w.%+-]+@[\w.%+-]+(\.[\w.%+-]+)+', email):
sendUpdateNotification({
'email': email,
'url': get_config().get('reports', 'urlRoot') + guid,
'status': reportData['status'],
})
newURL = get_config().get('reports', 'urlRoot') + guid
newURL += '?updated=' + str(int(random.uniform(0, 10000)))
newURL += '#secret=' + secret
start_response('302 Found', [('Location', newURL.encode('utf-8'))])
return []
def handleRequest(environ, start_response):
setupStderr(environ["wsgi.errors"])
if environ["REQUEST_METHOD"].upper() != "POST" or not environ.get("CONTENT_TYPE", "").startswith(
"application/x-www-form-urlencoded"
):
return showError("Unsupported request method", start_response)
try:
request_body_length = int(environ["CONTENT_LENGTH"])
except:
return showError("Invalid or missing Content-Length header", start_response)
request_body = environ["wsgi.input"].read(request_body_length)
params = {}
for key, value in parse_qsl(request_body):
params[key] = value.decode("utf-8")
guid = params.get("guid", "").lower()
if not re.match(r"^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$", guid):
return showError("Invalid or missing report GUID", start_response)
reportData = getReport(guid)
if reportData == None:
return showError("Report does not exist", start_response)
secret = calculateReportSecret(guid)
if params.get("secret", "") != secret and params.get("secret", "") != calculateReportSecret_compat(guid):
return showError("Wrong secret value", start_response)
reportData["status"] = params.get("status", "")
if len(reportData["status"]) > 1024:
reportData["status"] = reportData["status"][:1024]
oldusefulness = reportData.get("usefulness", "0")
reportData["usefulness"] = params.get("usefulness", "0")
if "email" in reportData:
updateUserUsefulness(getUserId(reportData["email"]), reportData["usefulness"], oldusefulness)
saveReport(guid, reportData)
if params.get("notify", "") and "email" in reportData:
email = reportData["email"]
email = re.sub(r" at ", r"@", email)
email = re.sub(r" dot ", r".", email)
if re.match(r"^[\w.%+-]+@[\w.%+-]+(\.[\w.%+-]+)+", email):
sendUpdateNotification(
{"email": email, "url": get_config().get("reports", "urlRoot") + guid, "status": reportData["status"]}
)
newURL = get_config().get("reports", "urlRoot") + guid
newURL += "?updated=" + str(int(random.uniform(0, 10000)))
newURL += "#secret=" + secret
start_response("302 Found", [("Location", newURL.encode("utf-8"))])
return []
def handleRequest(environ, start_response, params):
guid = params.get('guid', '').lower()
if not re.match(GUID_REGEX, guid):
return showError('Invalid or missing report GUID', start_response)
reportData = getReport(guid)
if reportData is None:
return showError('Report does not exist', start_response)
secret = calculateReportSecret(guid)
if (params.get('secret', '') != secret and
params.get('secret', '') != calculateReportSecret_compat(guid)):
return showError('Wrong secret value', start_response)
reportData['status'] = params.get('status', '')
if len(reportData['status']) > 1024:
reportData['status'] = reportData['status'][:1024]
oldusefulness = reportData.get('usefulness', '0')
reportData['usefulness'] = params.get('usefulness', '0')
if 'email' in reportData:
updateUserUsefulness(getUserId(reportData['email']),
reportData['usefulness'], oldusefulness)
saveReport(guid, reportData)
if params.get('notify', '') and 'email' in reportData:
email = reportData['email']
email = re.sub(r' at ', r'@', email)
email = re.sub(r' dot ', r'.', email)
if re.match(r'^[\w.%+-]+@[\w.%+-]+(\.[\w.%+-]+)+', email):
sendUpdateNotification({
'email':
email,
'url':
get_config().get('reports', 'urlRoot') + guid,
'status':
reportData['status'],
})
newURL = get_config().get('reports', 'urlRoot') + guid
newURL += '?updated=' + str(int(random.uniform(0, 10000)))
newURL += '#secret=' + secret
start_response('302 Found', [('Location', newURL.encode('utf-8'))])
return []
def handleRequest(environ, start_response, params):
guid = params.get('guid', '').lower()
if not re.match(GUID_REGEX, guid):
return showError('Invalid or missing report GUID', start_response)
reportData = getReport(guid)
if reportData is None:
return showError('Report does not exist', start_response)
secret = calculateReportSecret(guid)
if (params.get('secret', '') != secret and
params.get('secret', '') != calculateReportSecret_compat(guid)):
return showError('Wrong secret value', start_response)
reportData['status'] = params.get('status', '')
if len(reportData['status']) > 1024:
reportData['status'] = reportData['status'][:1024]
oldusefulness = reportData.get('usefulness', '0')
reportData['usefulness'] = params.get('usefulness', '0')
if 'email' in reportData:
updateUserUsefulness(getUserId(reportData['email']),
reportData['usefulness'], oldusefulness)
saveReport(guid, reportData)
if params.get('notify', '') and 'email' in reportData:
email = reportData['email']
email = re.sub(r' at ', r'@', email)
email = re.sub(r' dot ', r'.', email)
if re.match(r'^[\w.%+-]+@[\w.%+-]+(\.[\w.%+-]+)+', email):
sendUpdateNotification({
'email': email,
'url': get_config().get('reports', 'urlRoot') + guid,
'status': reportData['status'],
})
newURL = get_config().get('reports', 'urlRoot') + guid
newURL += '?updated=' + str(int(random.uniform(0, 10000)))
newURL += '#secret=' + secret
start_response('302 Found', [('Location', newURL.encode('utf-8'))])
return []
def scanReports():
global fakeSubscription, interval, subscriptions, startTime
result = []
for dbreport in getReports(startTime):
matchSubscriptions = {}
recipients = []
reportSubscriptions = getReportSubscriptions(dbreport['guid'])
if dbreport['type'] == 'false positive' or dbreport['type'] == 'false negative':
for subscription in reportSubscriptions:
subscriptionID = subscription.get('url', 'unknown')
if subscriptionID in subscriptions:
if subscription.get('hasmatches', 0) > 0:
matchSubscriptions[subscriptionID] = subscriptions[subscriptionID]
# Send false negatives to all subscription authors, false positives
# only to subscriptions with matching filters
if dbreport['type'] == 'false negative' or subscription.get('hasmatches', 0) > 0:
recipients.append(subscriptions[subscriptionID])
elif interval != 'week':
# Send type "other" to fake subscription - daily reports
recipients.append(fakeSubscription)
subscriptionID = fakeSubscription.get('url', 'unknown')
if len(recipients) == 0:
continue
report = {
'url': get_config().get('reports', 'urlRoot') + dbreport['guid'] + '#secret=' + calculateReportSecret(dbreport['guid']),
'weight': calculateReportWeight(dbreport, reportSubscriptions),
'site': dbreport['site'],
'subscriptions': recipients,
'comment': re.sub(r'[\x00-\x20]', r' ', dbreport['comment']) if dbreport['comment'] != None else '',
'type': dbreport['type'],
'numSubscriptions': len(reportSubscriptions),
'matchSubscriptions': matchSubscriptions.values(),
'contact': dbreport['contact'],
'hasscreenshot': dbreport['hasscreenshot'],
'knownIssues': dbreport['knownissues'],
}
result.append(report)
return result
def updateDigests(dir):
global currentTime
subs = subscriptionParser.readSubscriptions()
defname, defemail = parseaddr(get_config().get(
'reports', 'defaultSubscriptionRecipient'))
subscriptions = {}
emails = {}
emails[defemail] = []
for subscription in subs.values():
for title, url, complete in subscription.variants:
subscriptions[url] = subscription
name, email = parseaddr(subscription.email)
if email != '':
emails[email] = []
startTime = currentTime - get_config().getint('reports',
'digestDays') * 24 * 60 * 60
for dbreport in getReports(startTime):
report = {
'guid':
dbreport['guid'],
'status':
dbreport['status'],
'url':
get_config().get('reports', 'urlRoot') + dbreport['guid'] +
'#secret=' + calculateReportSecret(dbreport['guid']),
'site':
dbreport['site'],
'comment':
dbreport['comment'],
'type':
dbreport['type'],
'subscriptions': [],
'contact':
dbreport['contact'],
'score':
getUserUsefulnessScore(dbreport['contact']),
'hasscreenshot':
dbreport['hasscreenshot'],
'knownIssues':
dbreport['knownissues'],
'time':
dbreport['ctime'],
}
recipients = set()
reportSubscriptions = getReportSubscriptions(dbreport['guid'])
if dbreport['type'] == 'false positive' or dbreport[
'type'] == 'false negative':
for subscription in reportSubscriptions:
subscriptionID = subscription.get('url', 'unknown')
# Send false negatives to all subscription authors, false positives
# only to subscriptions with matching filters
if subscriptionID in subscriptions and (
dbreport['type'] == 'false negative'
or subscription.get('hasmatches', 0) > 0):
name, email = parseaddr(
subscriptions[subscriptionID].email)
if email and not email in recipients:
recipients.add(email)
emails[email].append(report)
report['subscriptions'].append(
getSubscriptionInfo(subscriptions[subscriptionID]))
else:
for subscription in reportSubscriptions:
subscriptionID = subscription.get('url', 'unknown')
report['subscriptions'].append(
getSubscriptionInfo(subscriptions[subscriptionID]))
recipients.add(defemail)
emails[defemail].append(report)
# Generate new digests
digests = set()
for email, reports in emails.iteritems():
if len(reports) == 0:
continue
file = getDigestPath(dir, email)
template = get_template(get_config().get('reports',
'htmlDigestTemplate'))
template.stream({
'email': email,
'reports': reports
}).dump(file, encoding='utf-8')
digests.add(file)
# Remove not updated digests which are more then 2 weeks old
for filename in os.listdir(dir):
file = os.path.join(dir, filename)
if os.path.isfile(file) and file not in digests and re.match(
r'^[\da-f]{32}\.html$', filename
) and os.stat(file).st_mtime < currentTime - 14 * 24 * 60 * 60:
os.remove(file)
def updateDigests(dir):
global currentTime
subs = subscriptionParser.readSubscriptions()
defname, defemail = parseaddr(get_config().get('reports', 'defaultSubscriptionRecipient'))
subscriptions = {}
emails = {}
emails[defemail] = []
for subscription in subs.values():
for title, url, complete in subscription.variants:
subscriptions[url] = subscription
name, email = parseaddr(subscription.email)
if email != '':
emails[email] = []
startTime = currentTime - get_config().getint('reports', 'digestDays') * 24*60*60
for dbreport in getReports(startTime):
report = {
'guid': dbreport['guid'],
'status': dbreport['status'],
'url': get_config().get('reports', 'urlRoot') + dbreport['guid'] + '#secret=' + calculateReportSecret(dbreport['guid']),
'site': dbreport['site'],
'comment': dbreport['comment'],
'type': dbreport['type'],
'subscriptions': [],
'contact': dbreport['contact'],
'score': getUserUsefulnessScore(dbreport['contact']),
'hasscreenshot': dbreport['hasscreenshot'],
'knownIssues': dbreport['knownissues'],
'time': dbreport['ctime'],
}
recipients = set()
reportSubscriptions = getReportSubscriptions(dbreport['guid'])
if dbreport['type'] == 'false positive' or dbreport['type'] == 'false negative':
for subscription in reportSubscriptions:
subscriptionID = subscription.get('url', 'unknown')
# Send false negatives to all subscription authors, false positives
# only to subscriptions with matching filters
if subscriptionID in subscriptions and (dbreport['type'] == 'false negative' or subscription.get('hasmatches', 0) > 0):
name, email = parseaddr(subscriptions[subscriptionID].email)
if email and not email in recipients:
recipients.add(email)
emails[email].append(report)
report['subscriptions'].append(getSubscriptionInfo(subscriptions[subscriptionID]))
else:
for subscription in reportSubscriptions:
subscriptionID = subscription.get('url', 'unknown')
report['subscriptions'].append(getSubscriptionInfo(subscriptions[subscriptionID]))
recipients.add(defemail)
emails[defemail].append(report)
# Generate new digests
digests = set()
for email, reports in emails.iteritems():
if len(reports) == 0:
continue
file = getDigestPath(dir, email)
template = get_template(get_config().get('reports', 'htmlDigestTemplate'))
template.stream({'email': email, 'reports': reports}).dump(file, encoding='utf-8')
digests.add(file)
# Remove not updated digests which are more then 2 weeks old
for filename in os.listdir(dir):
file = os.path.join(dir, filename)
if os.path.isfile(file) and file not in digests and re.match(r'^[\da-f]{32}\.html$', filename) and os.stat(file).st_mtime < currentTime - 14*24*60*60:
os.remove(file)
def updateDigests(dir):
global currentTime
subs = subscriptionParser.readSubscriptions()
defname, defemail = parseaddr(get_config().get("reports", "defaultSubscriptionRecipient"))
subscriptions = {}
emails = {}
emails[defemail] = []
for subscription in subs.values():
for title, url, complete in subscription.variants:
subscriptions[url] = subscription
name, email = parseaddr(subscription.email)
if email != "":
emails[email] = []
startTime = currentTime - get_config().getint("reports", "digestDays") * 24 * 60 * 60
for dbreport in getReports(startTime):
report = {
"guid": dbreport["guid"],
"status": dbreport["status"],
"url": get_config().get("reports", "urlRoot")
+ dbreport["guid"]
+ "#secret="
+ calculateReportSecret(dbreport["guid"]),
"site": dbreport["site"],
"comment": dbreport["comment"],
"type": dbreport["type"],
"subscriptions": [],
"contact": dbreport["contact"],
"score": getUserUsefulnessScore(dbreport["contact"]),
"hasscreenshot": dbreport["hasscreenshot"],
"knownIssues": dbreport["knownissues"],
"time": dbreport["ctime"],
}
recipients = set()
reportSubscriptions = getReportSubscriptions(dbreport["guid"])
if dbreport["type"] == "false positive" or dbreport["type"] == "false negative":
for subscription in reportSubscriptions:
subscriptionID = subscription.get("url", "unknown")
# Send false negatives to all subscription authors, false positives
# only to subscriptions with matching filters
if subscriptionID in subscriptions and (
dbreport["type"] == "false negative" or subscription.get("hasmatches", 0) > 0
):
name, email = parseaddr(subscriptions[subscriptionID].email)
if email and not email in recipients:
recipients.add(email)
emails[email].append(report)
report["subscriptions"].append(getSubscriptionInfo(subscriptions[subscriptionID]))
else:
for subscription in reportSubscriptions:
subscriptionID = subscription.get("url", "unknown")
report["subscriptions"].append(getSubscriptionInfo(subscriptions[subscriptionID]))
recipients.add(defemail)
emails[defemail].append(report)
# Generate new digests
digests = set()
for email, reports in emails.iteritems():
if len(reports) == 0:
continue
file = getDigestPath(dir, email)
template = get_template(get_config().get("reports", "htmlDigestTemplate"))
template.stream({"email": email, "reports": reports}).dump(file, encoding="utf-8")
digests.add(file)
# Remove not updated digests which are more then 2 weeks old
for filename in os.listdir(dir):
file = os.path.join(dir, filename)
if (
os.path.isfile(file)
and file not in digests
and re.match(r"^[\da-f]{32}\.html$", filename)
and os.stat(file).st_mtime < currentTime - 14 * 24 * 60 * 60
):
os.remove(file)
