def writeSubscriptions(templateName, outputFile=None):
subscriptions = subscriptionParser.readSubscriptions().values()
template = get_template(get_config().get('subscriptions',
templateName + 'Template'))
if outputFile == None:
outputFile = get_config().get('subscriptions', templateName + 'File')
template.stream({
'subscriptions': subscriptions
}).dump(outputFile, encoding='utf-8')
def checkSubscriptions():
subscriptions = subscriptionParser.readSubscriptions().values()
subscriptions.sort(key=lambda s: s.name.lower())
urls = {}
sites = {}
for subscription in subscriptions:
for key in ('homepage', 'forum', 'blog', 'faq', 'contact', 'changelog',
'policy'):
url = getattr(subscription, key)
if url != None:
urls[url] = True
for (title, url, complete) in subscription.variants:
urls[url] = True
pool = eventlet.GreenPool()
for (url, result) in pool.imap(checkURL, urls.iterkeys()):
urls[url] = result
if result is False:
sites[urlparse(url).netloc] = True
for (site, result) in pool.imap(checkSite, sites.iterkeys()):
sites[site] = result
result = []
for subscription in subscriptions:
s = {'name': subscription.name, 'links': []}
result.append(s)
for key in ('homepage', 'forum', 'blog', 'faq', 'contact', 'changelog',
'policy'):
url = getattr(subscription, key)
if url != None:
site = urlparse(url).netloc
s['links'].append({
'url': url,
'title': key[0].upper() + key[1:],
'result': urls[url],
'siteResult': site in sites and sites[site],
})
for (title, url, complete) in subscription.variants:
site = urlparse(url).netloc
s['links'].append({
'url': url,
'title': title,
'result': urls[url],
'siteResult': site in sites and sites[site],
})
return result
def checkSubscriptions():
subscriptions = subscriptionParser.readSubscriptions().values()
subscriptions.sort(key=lambda s: s.name.lower())
urls = {}
sites = {}
for subscription in subscriptions:
for key in ('homepage', 'forum', 'blog', 'faq', 'contact', 'changelog', 'policy'):
url = getattr(subscription, key)
if url != None:
urls[url] = True
for (title, url, complete) in subscription.variants:
urls[url] = True
pool = eventlet.GreenPool()
for (url, result) in pool.imap(checkURL, urls.iterkeys()):
urls[url] = result
if result is False:
sites[urlparse(url).netloc] = True
for (site, result) in pool.imap(checkSite, sites.iterkeys()):
sites[site] = result
result = []
for subscription in subscriptions:
s = {'name': subscription.name, 'links': []}
result.append(s)
for key in ('homepage', 'forum', 'blog', 'faq', 'contact', 'changelog', 'policy'):
url = getattr(subscription, key)
if url != None:
site = urlparse(url).netloc
s['links'].append({
'url': url,
'title': key[0].upper() + key[1:],
'result': urls[url],
'siteResult': site in sites and sites[site],
})
for (title, url, complete) in subscription.variants:
site = urlparse(url).netloc
s['links'].append({
'url': url,
'title': title,
'result': urls[url],
'siteResult': site in sites and sites[site],
})
return result
def loadSubscriptions():
global interval, weekDay
subscriptions = subscriptionParser.readSubscriptions()
results = {}
resultList = []
for subscription in subscriptions.values():
if subscription.digest == 'daily' and interval == 'week':
continue
if subscription.digest == 'weekly' and interval == 'day':
continue
if interval == 'week' and subscription.digestDay != weekDay:
continue
for [title, url, complete] in subscription.variants:
results[url] = subscription
resultList.append(subscription)
return (results, resultList)
def loadSubscriptions():
global interval, weekDay
subscriptions = subscriptionParser.readSubscriptions()
results = {}
resultList = []
for subscription in subscriptions.values():
if subscription.digest == 'daily' and interval == 'week':
continue
if subscription.digest == 'weekly' and interval == 'day':
continue
if interval == 'week' and subscription.digestDay != weekDay:
continue
for [title, url, complete] in subscription.variants:
results[url] = subscription
resultList.append(subscription)
return (results, resultList)
def updateSubscriptionList():
cursor = get_db().cursor(MySQLdb.cursors.DictCursor)
executeQuery(cursor, 'SELECT id, url FROM #PFX#subscriptions')
subids = {}
for dbsub in cursor:
subids[dbsub['url']] = dbsub['id']
subscriptions = subscriptionParser.readSubscriptions()
for subscription in subscriptions.values():
for title, url, complete in subscription.variants:
id = subids.get(url)
if id == None:
executeQuery(cursor, 'INSERT INTO #PFX#subscriptions (url) VALUES (%s)', url)
else:
del subids[url]
for url in subids:
executeQuery(cursor, 'DELETE FROM #PFX#subscriptions WHERE id = %s', subids[url])
get_db().commit()
def loadSubscriptions(counts):
global interval
subscriptions = subscriptionParser.readSubscriptions()
knownURLs = {}
for subscription in subscriptions.values():
for title, url, complete in subscription.variants:
knownURLs[url] = True
(redirectData, goneData) = subscriptionParser.getFallbackData()
redirects = processFile(redirectData, counts)
gone = processFile(goneData, counts)
unaccounted = filter(lambda url: counts[url] >= 10, counts.keys())
unaccounted.sort(key=lambda url: counts[url], reverse=True)
for i in range(0, len(unaccounted)):
url = unaccounted[i]
mark = ' [?]'
if url in knownURLs:
mark = ''
unaccounted[i] = '%5i %s%s' % (counts[url], url, mark)
return (redirects, gone, unaccounted)
def loadSubscriptions(counts):
global interval
subscriptions = subscriptionParser.readSubscriptions()
knownURLs = {}
for subscription in subscriptions.values():
for title, url, complete in subscription.variants:
knownURLs[url] = True
(redirectData, goneData) = subscriptionParser.getFallbackData()
redirects = processFile(redirectData, counts)
gone = processFile(goneData, counts)
unaccounted = filter(lambda url: counts[url] >= 10, counts.keys())
unaccounted.sort(key=lambda url: counts[url], reverse=True)
for i in range(0, len(unaccounted)):
url = unaccounted[i]
mark = ' [?]'
if url in knownURLs:
mark = ''
unaccounted[i] = '%5i %s%s' % (counts[url], url, mark)
return (redirects, gone, unaccounted)
def writeSubscriptions(templateName, outputFile=None):
subscriptions = subscriptionParser.readSubscriptions().values()
template = get_template(get_config().get('subscriptions', templateName + 'Template'))
if outputFile == None:
outputFile = get_config().get('subscriptions', templateName + 'File')
template.stream({'subscriptions': subscriptions}).dump(outputFile, encoding='utf-8')
def updateDigests(dir):
global currentTime
subs = subscriptionParser.readSubscriptions()
defname, defemail = parseaddr(get_config().get(
'reports', 'defaultSubscriptionRecipient'))
subscriptions = {}
emails = {}
emails[defemail] = []
for subscription in subs.values():
for title, url, complete in subscription.variants:
subscriptions[url] = subscription
name, email = parseaddr(subscription.email)
if email != '':
emails[email] = []
startTime = currentTime - get_config().getint('reports',
'digestDays') * 24 * 60 * 60
for dbreport in getReports(startTime):
report = {
'guid':
dbreport['guid'],
'status':
dbreport['status'],
'url':
get_config().get('reports', 'urlRoot') + dbreport['guid'] +
'#secret=' + calculateReportSecret(dbreport['guid']),
'site':
dbreport['site'],
'comment':
dbreport['comment'],
'type':
dbreport['type'],
'subscriptions': [],
'contact':
dbreport['contact'],
'score':
getUserUsefulnessScore(dbreport['contact']),
'hasscreenshot':
dbreport['hasscreenshot'],
'knownIssues':
dbreport['knownissues'],
'time':
dbreport['ctime'],
}
recipients = set()
reportSubscriptions = getReportSubscriptions(dbreport['guid'])
if dbreport['type'] == 'false positive' or dbreport[
'type'] == 'false negative':
for subscription in reportSubscriptions:
subscriptionID = subscription.get('url', 'unknown')
# Send false negatives to all subscription authors, false positives
# only to subscriptions with matching filters
if subscriptionID in subscriptions and (
dbreport['type'] == 'false negative'
or subscription.get('hasmatches', 0) > 0):
name, email = parseaddr(
subscriptions[subscriptionID].email)
if email and not email in recipients:
recipients.add(email)
emails[email].append(report)
report['subscriptions'].append(
getSubscriptionInfo(subscriptions[subscriptionID]))
else:
for subscription in reportSubscriptions:
subscriptionID = subscription.get('url', 'unknown')
report['subscriptions'].append(
getSubscriptionInfo(subscriptions[subscriptionID]))
recipients.add(defemail)
emails[defemail].append(report)
# Generate new digests
digests = set()
for email, reports in emails.iteritems():
if len(reports) == 0:
continue
file = getDigestPath(dir, email)
template = get_template(get_config().get('reports',
'htmlDigestTemplate'))
template.stream({
'email': email,
'reports': reports
}).dump(file, encoding='utf-8')
digests.add(file)
# Remove not updated digests which are more then 2 weeks old
for filename in os.listdir(dir):
file = os.path.join(dir, filename)
if os.path.isfile(file) and file not in digests and re.match(
r'^[\da-f]{32}\.html$', filename
) and os.stat(file).st_mtime < currentTime - 14 * 24 * 60 * 60:
os.remove(file)
def updateDigests(dir):
global currentTime
subs = subscriptionParser.readSubscriptions()
defname, defemail = parseaddr(get_config().get('reports', 'defaultSubscriptionRecipient'))
subscriptions = {}
emails = {}
emails[defemail] = []
for subscription in subs.values():
for title, url, complete in subscription.variants:
subscriptions[url] = subscription
name, email = parseaddr(subscription.email)
if email != '':
emails[email] = []
startTime = currentTime - get_config().getint('reports', 'digestDays') * 24*60*60
for dbreport in getReports(startTime):
report = {
'guid': dbreport['guid'],
'status': dbreport['status'],
'url': get_config().get('reports', 'urlRoot') + dbreport['guid'] + '#secret=' + calculateReportSecret(dbreport['guid']),
'site': dbreport['site'],
'comment': dbreport['comment'],
'type': dbreport['type'],
'subscriptions': [],
'contact': dbreport['contact'],
'score': getUserUsefulnessScore(dbreport['contact']),
'hasscreenshot': dbreport['hasscreenshot'],
'knownIssues': dbreport['knownissues'],
'time': dbreport['ctime'],
}
recipients = set()
reportSubscriptions = getReportSubscriptions(dbreport['guid'])
if dbreport['type'] == 'false positive' or dbreport['type'] == 'false negative':
for subscription in reportSubscriptions:
subscriptionID = subscription.get('url', 'unknown')
# Send false negatives to all subscription authors, false positives
# only to subscriptions with matching filters
if subscriptionID in subscriptions and (dbreport['type'] == 'false negative' or subscription.get('hasmatches', 0) > 0):
name, email = parseaddr(subscriptions[subscriptionID].email)
if email and not email in recipients:
recipients.add(email)
emails[email].append(report)
report['subscriptions'].append(getSubscriptionInfo(subscriptions[subscriptionID]))
else:
for subscription in reportSubscriptions:
subscriptionID = subscription.get('url', 'unknown')
report['subscriptions'].append(getSubscriptionInfo(subscriptions[subscriptionID]))
recipients.add(defemail)
emails[defemail].append(report)
# Generate new digests
digests = set()
for email, reports in emails.iteritems():
if len(reports) == 0:
continue
file = getDigestPath(dir, email)
template = get_template(get_config().get('reports', 'htmlDigestTemplate'))
template.stream({'email': email, 'reports': reports}).dump(file, encoding='utf-8')
digests.add(file)
# Remove not updated digests which are more then 2 weeks old
for filename in os.listdir(dir):
file = os.path.join(dir, filename)
if os.path.isfile(file) and file not in digests and re.match(r'^[\da-f]{32}\.html$', filename) and os.stat(file).st_mtime < currentTime - 14*24*60*60:
os.remove(file)
def updateDigests(dir):
global currentTime
subs = subscriptionParser.readSubscriptions()
defname, defemail = parseaddr(get_config().get("reports", "defaultSubscriptionRecipient"))
subscriptions = {}
emails = {}
emails[defemail] = []
for subscription in subs.values():
for title, url, complete in subscription.variants:
subscriptions[url] = subscription
name, email = parseaddr(subscription.email)
if email != "":
emails[email] = []
startTime = currentTime - get_config().getint("reports", "digestDays") * 24 * 60 * 60
for dbreport in getReports(startTime):
report = {
"guid": dbreport["guid"],
"status": dbreport["status"],
"url": get_config().get("reports", "urlRoot")
+ dbreport["guid"]
+ "#secret="
+ calculateReportSecret(dbreport["guid"]),
"site": dbreport["site"],
"comment": dbreport["comment"],
"type": dbreport["type"],
"subscriptions": [],
"contact": dbreport["contact"],
"score": getUserUsefulnessScore(dbreport["contact"]),
"hasscreenshot": dbreport["hasscreenshot"],
"knownIssues": dbreport["knownissues"],
"time": dbreport["ctime"],
}
recipients = set()
reportSubscriptions = getReportSubscriptions(dbreport["guid"])
if dbreport["type"] == "false positive" or dbreport["type"] == "false negative":
for subscription in reportSubscriptions:
subscriptionID = subscription.get("url", "unknown")
# Send false negatives to all subscription authors, false positives
# only to subscriptions with matching filters
if subscriptionID in subscriptions and (
dbreport["type"] == "false negative" or subscription.get("hasmatches", 0) > 0
):
name, email = parseaddr(subscriptions[subscriptionID].email)
if email and not email in recipients:
recipients.add(email)
emails[email].append(report)
report["subscriptions"].append(getSubscriptionInfo(subscriptions[subscriptionID]))
else:
for subscription in reportSubscriptions:
subscriptionID = subscription.get("url", "unknown")
report["subscriptions"].append(getSubscriptionInfo(subscriptions[subscriptionID]))
recipients.add(defemail)
emails[defemail].append(report)
# Generate new digests
digests = set()
for email, reports in emails.iteritems():
if len(reports) == 0:
continue
file = getDigestPath(dir, email)
template = get_template(get_config().get("reports", "htmlDigestTemplate"))
template.stream({"email": email, "reports": reports}).dump(file, encoding="utf-8")
digests.add(file)
# Remove not updated digests which are more then 2 weeks old
for filename in os.listdir(dir):
file = os.path.join(dir, filename)
if (
os.path.isfile(file)
and file not in digests
and re.match(r"^[\da-f]{32}\.html$", filename)
and os.stat(file).st_mtime < currentTime - 14 * 24 * 60 * 60
):
os.remove(file)