def test_get_template(tmpdir, mode):
"""Load template using relative or absolute path."""
template_path = tmpdir.join('template.tmpl')
template_path.write('value = {{ value }}')
if mode == 'absolute':
template = get_template(template_path.strpath)
else:
template = get_template('template.tmpl', template_path=tmpdir.strpath)
assert template.render({'value': 1}) == 'value = 1'
def make_handler(name, config):
try:
url = config['url'].value
except (KeyError, AttributeError):
raise Exception('No URL configured for form handler: ' + name)
try:
template = config['template'].value
get_template(template, autoescape=False)
except (KeyError, AttributeError):
raise Exception('No template configured for form handler: ' + name)
except jinja2.TemplateNotFound:
raise Exception('Template not found at: ' + template)
try:
fields = config['fields']
for field, spec in fields.items():
spec.value = {s.strip() for s in spec.value.split(',')}
except KeyError:
raise Exception('No fields configured for form handler: ' + name)
if len(fields) == 0:
raise Exception('No fields configured for form handler: ' + name)
@form_handler
def handler(environ, start_response, params):
response_headers = [('Content-Type', 'text/plain; charset=utf-8')]
errors = validate_fields(fields, params)
if errors:
start_response('400 Bad Request', response_headers)
return '\n'.join(errors)
time = datetime.datetime.now()
template_args = {
'time': time,
'fields': {field: params.get(field, '')
for field in fields},
}
try:
sendMail(template, template_args)
except:
print(traceback.print_exc(), file=sys.stderr)
start_response('500 Server Error', response_headers)
return ''
finally:
if 'csv_log' in config:
params = {
field: params.get(field, '').encode('utf8')
for field in fields
}
params['time'] = time
log_formdata(params, config['csv_log'].value)
start_response('200 OK', response_headers)
return ''
return url, handler
def writeUpdateManifest(self):
"""
Writes update manifest for the current build
"""
baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
if self.config.type == 'safari':
manifestPath = os.path.join(baseDir, 'updates.plist')
templateName = 'safariUpdateManifest'
autoescape = True
elif self.config.type == 'android':
manifestPath = os.path.join(baseDir, 'updates.xml')
templateName = 'androidUpdateManifest'
autoescape = True
else:
return
if not os.path.exists(baseDir):
os.makedirs(baseDir)
# ABP for Android used to have its own update manifest format. We need to
# generate both that and the new one in the libadblockplus format as long
# as a significant amount of users is on an old version.
if self.config.type == 'android':
newManifestPath = os.path.join(baseDir, 'update.json')
writeAndroidUpdateManifest(newManifestPath, [{
'basename': self.basename,
'version': self.version,
'updateURL': self.updateURL,
}])
template = get_template(get_config().get('extensions', templateName),
autoescape=autoescape)
template.stream({'extensions': [self]}).dump(manifestPath)
def sitekey_frame(environ, start_response):
template_path, template_file = os.path.split(
get_config().get('testpages', 'sitekeyFrameTemplate'),
)
template = get_template(template_file, template_path=template_path)
http_path = request_path(environ)
http_host = environ['HTTP_HOST']
http_ua = environ['HTTP_USER_AGENT']
key = M2Crypto.EVP.load_key(get_config().get('testpages', 'sitekeyPath'))
key.sign_init()
key.sign_update('\x00'.join([http_path, http_host, http_ua]))
public_key = base64.b64encode(key.as_der())
signature = base64.b64encode(key.final())
start_response('200 OK', [
('Content-Type', 'text/html; charset=utf-8'),
('X-Adblock-Key', '%s_%s' % (public_key, signature)),
])
return [template.render({
'public_key': public_key,
'signature': signature,
'http_path': http_path,
'http_host': http_host,
'http_ua': http_ua,
}).encode('utf-8')]
def updateIndex(self, versions):
"""
Updates index page listing all existing versions
"""
baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
if not os.path.exists(baseDir):
os.makedirs(baseDir)
outputFile = 'index.html'
outputPath = os.path.join(baseDir, outputFile)
links = []
for version in versions:
packageFile = self.basename + '-' + version + self.config.packageSuffix
changelogFile = self.basename + '-' + version + '.changelog.xhtml'
if not os.path.exists(os.path.join(baseDir, packageFile)):
# Oops
continue
link = {
'version': version,
'download': packageFile,
'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)),
'size': os.path.getsize(os.path.join(baseDir, packageFile)),
}
if os.path.exists(os.path.join(baseDir, changelogFile)):
link['changelog'] = changelogFile
links.append(link)
template = get_template(get_config().get('extensions', 'nightlyIndexPage'))
template.stream({'config': self.config, 'links': links}).dump(outputPath)
def write(self):
template = get_template(self.repo.padTemplate)
basename = self.repo.basename
filename = basename + '.xml'
pad = template.render({
'name': self.repo.name,
'type': self.repo.type,
'basename': basename,
'browser_name': self.browser_name,
'browser_min_version': self.browser_min_version,
'version': self.version,
'release_date': self.release_date,
'release_status': self.release_status,
'os_support': ','.join(self.os_support),
'language': ','.join(self.languages),
'download_size': self.download_size,
'download_url': self.download_url,
'pad_url': urljoin(self.repo.padURL, filename),
}).encode('utf-8')
path = os.path.join(self.repo.padDirectory, filename)
validate_pad(pad, path)
with open(path, 'wb') as file:
file.write(pad)
def writeUpdateManifest(self):
"""
Writes update.rdf file for the current build
"""
baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
if not os.path.exists(baseDir):
os.makedirs(baseDir)
if self.config.type == 'chrome' or self.config.type == 'opera':
manifestPath = os.path.join(baseDir, "updates.xml")
templateName = 'chromeUpdateManifest'
elif self.config.type == 'safari':
manifestPath = os.path.join(baseDir, "updates.plist")
templateName = 'safariUpdateManifest'
elif self.config.type == 'android':
manifestPath = os.path.join(baseDir, "updates.xml")
templateName = 'androidUpdateManifest'
# ABP for Android used to have its own update manifest format. We need to
# generate both that and the new one in the libadblockplus format as long
# as a significant amount of users is on an old version.
newManifestPath = os.path.join(baseDir, "update.json")
writeAndroidUpdateManifest(newManifestPath,
[{
'basename': self.basename,
'version': self.version,
'updateURL': self.updateURL
}])
else:
manifestPath = os.path.join(baseDir, "update.rdf")
templateName = 'geckoUpdateManifest'
template = get_template(get_config().get('extensions', templateName))
template.stream({'extensions': [self]}).dump(manifestPath)
def writeUpdateManifest(links):
"""
writes an update manifest for all extensions and Android apps
"""
extensions = {'gecko': [], 'android': [], 'safari': [], 'ie': []}
for repo in Configuration.getRepositoryConfigurations():
if repo.type not in extensions or not links.has_section(repo.repositoryName):
continue
data = readMetadata(repo, links.get(repo.repositoryName, 'version'))
data['updateURL'] = links.get(repo.repositoryName, 'downloadURL')
if data['updateURL'].startswith(repo.downloadsURL):
data['updateURL'] += "?update"
extensions[repo.type].append(data)
if len(extensions['android']) > 1:
print >>sys.stderr, 'Warning: more than one Android app defined, update manifest only works for one'
for repoType in extensions.iterkeys():
manifestPath = get_config().get('extensions', '%sUpdateManifestPath' % repoType)
if repoType == 'ie':
writeIEUpdateManifest(manifestPath, extensions[repoType])
else:
# ABP for Android used to have its own update manifest format. We need to
# generate both that and the new one in the libadblockplus format as long
# as a significant amount of users is on an old version.
if repoType == 'android':
newManifestPath = get_config().get("extensions",
"androidNewUpdateManifestPath")
writeAndroidUpdateManifest(newManifestPath, extensions[repoType])
template = get_template(get_config().get('extensions', '%sUpdateManifest' % repoType))
template.stream({'extensions': extensions[repoType]}).dump(manifestPath)
def write(self):
template = get_template(self.repo.padTemplate)
basename = self.repo.basename
filename = basename + '.xml'
pad = template.render({
'name': self.repo.name,
'type': self.repo.type,
'basename': basename,
'browser_name': self.browser_name,
'browser_min_version': self.browser_min_version,
'version': self.version,
'release_date': self.release_date,
'release_status': self.release_status,
'os_support': ','.join(self.os_support),
'language': ','.join(self.languages),
'download_size': self.download_size,
'download_url': self.download_url,
'pad_url': urljoin(self.repo.padURL, filename),
}).encode('utf-8')
path = os.path.join(self.repo.padDirectory, filename)
validate_pad(pad, path)
with open(path, 'wb') as file:
file.write(pad)
def sitekey_frame(environ, start_response):
template_path, template_file = os.path.split(
get_config().get('testpages', 'sitekeyFrameTemplate'), )
template = get_template(template_file, template_path=template_path)
key = M2Crypto.EVP.load_key(get_config().get('testpages', 'sitekeyPath'))
key.sign_init()
key.sign_update('\x00'.join((
request_path(environ),
environ['HTTP_HOST'],
environ['HTTP_USER_AGENT'],
)))
public_key = base64.b64encode(key.as_der())
signature = base64.b64encode(key.final())
start_response('200 OK',
[('Content-Type', 'text/html; charset=utf-8'),
('X-Adblock-Key', '%s_%s' % (public_key, signature))])
return [
template.render({
'public_key': public_key,
'signature': signature
}).encode('utf-8')
]
def handleRequest(environ, start_response):
if not environ.get('HTTP_X_ADBLOCK_PLUS'):
return showError('Please use Adblock Plus to submit reports',
start_response)
if environ['REQUEST_METHOD'].upper() != 'POST' or not environ.get(
'CONTENT_TYPE', '').startswith('text/xml'):
return showError('Unsupported request method', start_response)
params = parse_qs(environ.get('QUERY_STRING', ''))
requestVersion = params.get('version', ['0'])[0]
if requestVersion != '1':
return showError('Unsupported request version', start_response)
guid = params.get('guid', [''])[0].lower()
if not re.match(
r'^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$',
guid):
return showError('Invalid or missing GUID', start_response)
path = os.path.join(get_config().get('reports', 'dataPath'), guid + '.xml')
if os.path.exists(path) or os.path.exists(path + '.tmp'):
return showError('Duplicate GUID', start_response)
try:
request_size = int(environ['CONTENT_LENGTH'])
except (KeyError, ValueError):
return showError('Invalid or missing Content-Length header',
start_response, '411 Length Required')
dir = os.path.dirname(path)
if not os.path.exists(dir):
os.makedirs(dir)
try:
file = open(path + '.tmp', 'wb')
data = environ['wsgi.input'].read(request_size)
file.write(data)
file.close()
knownIssues = knownIssuesParser.findMatches(
data.splitlines(),
params.get('lang', ['en-US'])[0])
os.rename(path + '.tmp', path)
except Exception as e:
if os.path.isfile(path + '.tmp'):
os.remove(path + '.tmp')
raise e
template = get_template(get_config().get('reports',
'submitResponseTemplate'))
start_response('200 OK',
[('Content-Type', 'application/xhtml+xml; charset=utf-8')])
return [
template.render({
'url': get_config().get('reports', 'urlRoot') + guid,
'knownIssues': knownIssues
}).encode('utf-8')
]
def writeUpdateManifest(self):
"""
Writes update.rdf file for the current build
"""
baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
if self.config.type == 'safari':
manifestPath = os.path.join(baseDir, 'updates.plist')
templateName = 'safariUpdateManifest'
elif self.config.type == 'android':
manifestPath = os.path.join(baseDir, 'updates.xml')
templateName = 'androidUpdateManifest'
else:
return
if not os.path.exists(baseDir):
os.makedirs(baseDir)
# ABP for Android used to have its own update manifest format. We need to
# generate both that and the new one in the libadblockplus format as long
# as a significant amount of users is on an old version.
if self.config.type == 'android':
newManifestPath = os.path.join(baseDir, 'update.json')
writeAndroidUpdateManifest(newManifestPath, [{
'basename': self.basename,
'version': self.version,
'updateURL': self.updateURL
}])
template = get_template(get_config().get('extensions', templateName))
template.stream({'extensions': [self]}).dump(manifestPath)
def updateIndex(self, versions):
"""
Updates index page listing all existing versions
"""
baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
if not os.path.exists(baseDir):
os.makedirs(baseDir)
outputFile = "index.html"
outputPath = os.path.join(baseDir, outputFile)
links = []
for version in versions:
packageFile = self.basename + '-' + version + self.config.packageSuffix
changelogFile = self.basename + '-' + version + '.changelog.xhtml'
if not os.path.exists(os.path.join(baseDir, packageFile)):
# Oops
continue
link = {
'version': version,
'download': packageFile,
'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)),
'size': os.path.getsize(os.path.join(baseDir, packageFile))
}
if os.path.exists(os.path.join(baseDir, changelogFile)):
link['changelog'] = changelogFile
links.append(link)
template = get_template(get_config().get('extensions', 'nightlyIndexPage'))
template.stream({'config': self.config, 'links': links}).dump(outputPath)
def writeChangelog(self, changes):
"""
write the changelog file into the cloned repository
"""
baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
if not os.path.exists(baseDir):
os.makedirs(baseDir)
changelogFile = "%s-%s.changelog.xhtml" % (self.basename, self.version)
changelogPath = os.path.join(baseDir, changelogFile)
self.changelogURL = urlparse.urljoin(
self.config.nightliesURL, self.basename + '/' + changelogFile)
template = get_template(get_config().get('extensions',
'changelogTemplate'))
template.stream({
'changes': changes
}).dump(changelogPath, encoding='utf-8')
linkPath = os.path.join(baseDir, '00latest.changelog.xhtml')
if hasattr(os, 'symlink'):
if os.path.exists(linkPath):
os.remove(linkPath)
os.symlink(os.path.basename(changelogPath), linkPath)
else:
shutil.copyfile(changelogPath, linkPath)
def _generate_comments(repository_name, changes_by_issue):
comments = {}
template = get_template("hg/template/issue_commit_comment.tmpl",
autoescape=False)
for issue_id, changes in changes_by_issue.iteritems():
comments[issue_id] = template.render({"repository_name": repository_name,
"changes": changes})
return comments
def writeSubscriptions(templateName, outputFile=None):
subscriptions = subscriptionParser.readSubscriptions().values()
template = get_template(get_config().get('subscriptions',
templateName + 'Template'))
if outputFile == None:
outputFile = get_config().get('subscriptions', templateName + 'File')
template.stream({
'subscriptions': subscriptions
}).dump(outputFile, encoding='utf-8')
def saveReport(guid, reportData, isNew=False):
cursor = get_db().cursor()
screenshot = reportData.get('screenshot', None)
if screenshot != None:
reportData['hasscreenshot'] = 2 if reportData.get('screenshotEdited', False) else 1
try:
saveScreenshot(guid, screenshot)
except (TypeError, UnicodeEncodeError):
reportData['hasscreenshot'] = 0
del reportData['screenshot']
knownIssues = len(reportData.get('knownIssues', []))
contact = getUserId(reportData.get('email', None)) if reportData.get('email', None) else None
dumpstr = marshal.dumps(reportData)
if contact != None and isNew:
executeQuery(cursor, 'INSERT INTO #PFX#users (id, reports) VALUES (%s, 1) ON DUPLICATE KEY UPDATE reports = reports + 1', contact)
executeQuery(cursor,
'''INSERT INTO #PFX#reports (guid, type, ctime, site, comment, status, contact, hasscreenshot, knownissues, dump)
VALUES (%(guid)s, %(type)s, FROM_UNIXTIME(%(ctime)s), %(site)s, %(comment)s, %(status)s, %(contact)s,
%(hasscreenshot)s, %(knownissues)s, _binary %(dump)s) ON DUPLICATE KEY
UPDATE type = %(type)s, site = %(site)s, comment = %(comment)s, status = %(status)s,
hasscreenshot = %(hasscreenshot)s, knownissues = %(knownissues)s, dump = _binary %(dump)s''',
{'guid': guid, 'type': reportData.get('type', None), 'ctime': reportData['time'], 'site': reportData.get('siteName', None),
'comment': reportData.get('comment', None), 'status': reportData.get('status', None), 'contact': contact,
'hasscreenshot': reportData.get('hasscreenshot', 0), 'knownissues': knownIssues, 'dump': dumpstr})
if len(reportData['subscriptions']) > 0:
for sn in reportData['subscriptions']:
executeQuery(cursor, 'SELECT id FROM #PFX#subscriptions WHERE url = %s', sn['id'])
id = cursor.fetchone()
if id != None:
def filterMatch(f):
return any(u == sn['id'] for u in f.get('subscriptions', []))
hasMatches = any(filterMatch(f) for f in reportData.get('filters', []))
executeQuery(cursor, 'INSERT IGNORE INTO #PFX#sublists (report, list, hasmatches) VALUES (%s, %s, %s)', (guid, id[0], hasMatches))
get_db().commit()
reportData['guid'] = guid
if contact:
# TODO: The mail anonymization should happen in the template, not here
origEmail = reportData['email']
email = reportData['email']
email = re.sub(r' at ', r'@', email)
email = re.sub(r' dot ', r'.', email)
reportData['email'] = anonymizeMail(email)
reportData['uid'] = contact
file = os.path.join(get_config().get('reports', 'dataPath'), guid[0], guid[1], guid[2], guid[3], guid + '.html')
dir = os.path.dirname(file)
if not os.path.exists(dir):
os.makedirs(dir)
template = get_template(get_config().get('reports', 'webTemplate'))
template.stream(reportData).dump(file, encoding='utf-8')
if contact:
reportData['email'] = origEmail
def _post_comments(ui, repo, config, refs):
repo_name = posixpath.split(repo.url())[1]
template = get_template('hg/template/issue_commit_comment.tmpl',
autoescape=False)
for ref in refs:
comment_text = template.render({
'repository_name': repo_name,
'changes': ref.commits,
'format_description': _format_description,
})
with _trac_proxy(ui, config, 'getting issue {}'.format(ref.id)) as tp:
attrs = tp.ticket.get(ref.id)[3]
changes = {'_ts': attrs['_ts'], 'action': 'leave'}
_update_issue(ui, config, ref.id, changes, comment_text)
def _generate_comments(repository_name, changes_by_issue):
comments = {}
template = get_template('hg/template/issue_commit_comment.tmpl',
autoescape=False)
for issue_id, changes in changes_by_issue.items():
comments[issue_id] = template.render({
'repository_name':
repository_name,
'changes':
changes,
'format_description':
_format_description,
})
return comments
def sitekey_frame(environ, start_response):
template_path, template_file = os.path.split(get_config().get("testpages", "sitekeyFrameTemplate"))
template = get_template(template_file, template_path=template_path)
key = M2Crypto.EVP.load_key(get_config().get("testpages", "sitekeyPath"))
key.sign_init()
key.sign_update("\x00".join((request_path(environ), environ["HTTP_HOST"], environ["HTTP_USER_AGENT"])))
public_key = base64.b64encode(key.as_der())
signature = base64.b64encode(key.final())
start_response(
"200 OK", [("Content-Type", "text/html; charset=utf-8"), ("X-Adblock-Key", "%s_%s" % (public_key, signature))]
)
return [template.render({"public_key": public_key, "signature": signature}).encode("utf-8")]
def handleRequest(environ, start_response):
if not environ.get('HTTP_X_ADBLOCK_PLUS'):
return showError('Please use Adblock Plus to submit reports', start_response)
if environ['REQUEST_METHOD'].upper() != 'POST' or not environ.get('CONTENT_TYPE', '').startswith('text/xml'):
return showError('Unsupported request method', start_response)
params = parse_qs(environ.get('QUERY_STRING', ''))
requestVersion = params.get('version', ['0'])[0]
if requestVersion != '1':
return showError('Unsupported request version', start_response)
guid = params.get('guid', [''])[0].lower()
if not re.match(r'^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$', guid):
return showError('Invalid or missing GUID', start_response)
path = os.path.join(get_config().get('reports', 'dataPath'), guid + '.xml')
if os.path.exists(path) or os.path.exists(path + '.tmp'):
return showError('Duplicate GUID', start_response)
try:
request_size = int(environ['CONTENT_LENGTH'])
except (KeyError, ValueError):
return showError('Invalid or missing Content-Length header', start_response,
'411 Length Required')
dir = os.path.dirname(path)
if not os.path.exists(dir):
os.makedirs(dir)
try:
file = open(path + '.tmp', 'wb')
data = environ['wsgi.input'].read(request_size)
file.write(data)
file.close()
knownIssues = knownIssuesParser.findMatches(data.splitlines(), params.get('lang', ['en-US'])[0])
os.rename(path + '.tmp', path)
except Exception as e:
if os.path.isfile(path + '.tmp'):
os.remove(path + '.tmp')
raise e
template = get_template(get_config().get('reports', 'submitResponseTemplate'))
start_response('200 OK', [('Content-Type', 'application/xhtml+xml; charset=utf-8')])
return [template.render({'url': get_config().get('reports', 'urlRoot') + guid, 'knownIssues': knownIssues}).encode('utf-8')]
def _post_comments(ui, repo, config, refs):
repo_name = posixpath.split(repo.url())[1]
template = get_template('hg/template/issue_commit_comment.tmpl',
autoescape=False)
for ref in refs:
comment_text = template.render({
'repository_name':
repo_name,
'changes':
ref.commits,
'format_description':
_format_description,
})
with _trac_proxy(ui, config, 'getting issue {}'.format(ref.id)) as tp:
attrs = tp.ticket.get(ref.id)[3]
changes = {'_ts': attrs['_ts'], 'action': 'leave'}
_update_issue(ui, config, ref.id, changes, comment_text)
def handleRequest(environ, start_response):
setupStderr(environ['wsgi.errors'])
params = parse_qs(environ.get('QUERY_STRING', ''))
id = params.get('id', [''])[0].lower()
if not re.match(r'^[\da-f]{32}$', id):
return showError('Invalid or missing ID', start_response)
user = getUser(id)
if user == None:
return showError('User not found', start_response)
user['reportlist'] = getReportsForUser(id)
template = get_template(get_config().get('reports', 'showUserTemplate'))
start_response('200 OK', [('Content-Type', 'application/xhtml+xml; charset=utf-8')])
return [template.render(user).encode('utf-8')]
def handleRequest(environ, start_response):
setupStderr(environ['wsgi.errors'])
params = parse_qs(environ.get('QUERY_STRING', ''))
id = params.get('id', [''])[0].lower()
if not re.match(r'^[\da-f]{32}$', id):
return showError('Invalid or missing ID', start_response)
user = getUser(id)
if user == None:
return showError('User not found', start_response)
user['reportlist'] = getReportsForUser(id)
template = get_template(get_config().get('reports', 'showUserTemplate'))
start_response('200 OK',
[('Content-Type', 'application/xhtml+xml; charset=utf-8')])
return [template.render(user).encode('utf-8')]
def writeChangelog(self, changes):
"""
write the changelog file into the cloned repository
"""
baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
if not os.path.exists(baseDir):
os.makedirs(baseDir)
changelogFile = "%s-%s.changelog.xhtml" % (self.basename, self.version)
changelogPath = os.path.join(baseDir, changelogFile)
self.changelogURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '/' + changelogFile)
template = get_template(get_config().get('extensions', 'changelogTemplate'))
template.stream({'changes': changes}).dump(changelogPath)
linkPath = os.path.join(baseDir, '00latest.changelog.xhtml')
if hasattr(os, 'symlink'):
if os.path.exists(linkPath):
os.remove(linkPath)
os.symlink(os.path.basename(changelogPath), linkPath)
else:
shutil.copyfile(changelogPath, linkPath)
def writeUpdateManifest(links):
"""
writes an update manifest for all extensions and Android apps
"""
extensions = {'android': [], 'safari': [], 'ie': []}
for repo in Configuration.getRepositoryConfigurations():
if repo.type not in extensions or not links.has_section(
repo.repositoryName):
continue
data = readMetadata(repo, links.get(repo.repositoryName, 'version'))
data['updateURL'] = links.get(repo.repositoryName, 'downloadURL')
if data['updateURL'].startswith(repo.downloadsURL):
data['updateURL'] += '?update'
extensions[repo.type].append(data)
if len(extensions['android']) > 1:
print >> sys.stderr, 'Warning: more than one Android app defined, update manifest only works for one'
for repoType in extensions.iterkeys():
manifestPath = get_config().get('extensions',
'%sUpdateManifestPath' % repoType)
if repoType == 'ie':
writeIEUpdateManifest(manifestPath, extensions[repoType])
else:
# ABP for Android used to have its own update manifest format. We need to
# generate both that and the new one in the libadblockplus format as long
# as a significant amount of users is on an old version.
if repoType == 'android':
newManifestPath = get_config().get(
'extensions', 'androidNewUpdateManifestPath')
writeAndroidUpdateManifest(newManifestPath,
extensions[repoType])
path = get_config().get('extensions',
'%sUpdateManifest' % repoType)
template = get_template(path,
autoescape=not path.endswith('.json'))
template.stream({
'extensions': extensions[repoType]
}).dump(manifestPath)
def writeUpdateManifest(links):
"""
writes an update manifest for all Gecko extensions and Android apps
"""
extensions = {'gecko': [], 'android': []}
for repo in Configuration.getRepositoryConfigurations():
if repo.type not in extensions or not links.has_section(repo.repositoryName):
continue
data = readMetadata(repo, links.get(repo.repositoryName, 'version'))
data['updateURL'] = links.get(repo.repositoryName, 'downloadURL')
if data['updateURL'].startswith(repo.downloadsURL):
data['updateURL'] += "?update"
extensions[repo.type].append(data)
if len(extensions['android']) > 1:
print >>sys.stderr, 'Warning: more than one Android app defined, update manifest only works for one'
for repoType in extensions.iterkeys():
manifestPath = get_config().get('extensions', '%sUpdateManifestPath' % repoType)
template = get_template(get_config().get('extensions', '%sUpdateManifest' % repoType))
template.stream({'extensions': extensions[repoType]}).dump(manifestPath)
def writeUpdateManifest(self):
"""
Writes update.rdf file for the current build
"""
baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
if not os.path.exists(baseDir):
os.makedirs(baseDir)
if self.config.type == 'chrome' or self.config.type == 'opera':
manifestPath = os.path.join(baseDir, "updates.xml")
templateName = 'chromeUpdateManifest'
elif self.config.type == 'safari':
manifestPath = os.path.join(baseDir, "updates.plist")
templateName = 'safariUpdateManifest'
elif self.config.type == 'android':
manifestPath = os.path.join(baseDir, "updates.xml")
templateName = 'androidUpdateManifest'
else:
manifestPath = os.path.join(baseDir, "update.rdf")
templateName = 'geckoUpdateManifest'
template = get_template(get_config().get('extensions', templateName))
template.stream({'extensions': [self]}).dump(manifestPath)
def sitekey_frame(environ, start_response):
template_path, template_file = os.path.split(get_config().get(
"testpages", "sitekeyFrameTemplate"))
template = get_template(template_file, template_path=template_path)
key = M2Crypto.EVP.load_key(get_config().get("testpages", "sitekeyPath"))
key.sign_init()
key.sign_update("\0".join((request_path(environ), environ["HTTP_HOST"],
environ["HTTP_USER_AGENT"])))
public_key = base64.b64encode(key.as_der())
signature = base64.b64encode(key.final())
start_response("200 OK",
[("Content-Type", "text/html; charset=utf-8"),
("X-Adblock-Key", "%s_%s" % (public_key, signature))])
return [
template.render({
"public_key": public_key,
"signature": signature
}).encode("utf-8")
]
request_size= int(environ['CONTENT_LENGTH'])
except (KeyError, ValueError):
return showError('Invalid or missing Content-Length header', start_response,
'411 Length Required')
dir = os.path.dirname(path)
if not os.path.exists(dir):
os.makedirs(dir)
try:
file = open(path + '.tmp', 'wb')
data = environ['wsgi.input'].read(request_size)
file.write(data)
file.close()
knownIssues = knownIssuesParser.findMatches(data.splitlines(), params.get('lang', ['en-US'])[0])
os.rename(path + '.tmp', path);
except Exception, e:
if os.path.isfile(path + '.tmp'):
os.remove(path + '.tmp')
raise e
template = get_template(get_config().get('reports', 'submitResponseTemplate'))
start_response('200 OK', [('Content-Type', 'application/xhtml+xml; charset=utf-8')])
return [template.render({'url': get_config().get('reports', 'urlRoot') + guid, 'knownIssues': knownIssues}).encode('utf-8')]
def showError(message, start_response, response_code='400 Processing Error'):
template = get_template(get_config().get('reports', 'errorTemplate'))
start_response(response_code, [('Content-Type', 'application/xhtml+xml; charset=utf-8')])
return [template.render({'message': message}).encode('utf-8')]
def writeSubscriptions(templateName, outputFile=None):
subscriptions = subscriptionParser.readSubscriptions().values()
template = get_template(get_config().get('subscriptions', templateName + 'Template'))
if outputFile == None:
outputFile = get_config().get('subscriptions', templateName + 'File')
template.stream({'subscriptions': subscriptions}).dump(outputFile, encoding='utf-8')
if not re.match(r'^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$', guid):
return showError('Invalid or missing GUID', start_response)
path = os.path.join(get_config().get('reports', 'dataPath'), guid + '.xml')
if os.path.exists(path) or os.path.exists(path + '.tmp'):
return showError('Duplicate GUID', start_response)
dir = os.path.dirname(path)
if not os.path.exists(dir):
os.makedirs(dir)
try:
file = open(path + '.tmp', 'wb')
iter = dataIterator(environ['wsgi.input'], file)
knownIssues = knownIssuesParser.findMatches(iter, params.get('lang', ['en-US'])[0])
file.close()
os.rename(path + '.tmp', path);
except Exception, e:
if os.path.isfile(path + '.tmp'):
os.remove(path + '.tmp')
raise e
template = get_template(get_config().get('reports', 'submitResponseTemplate'))
start_response('200 OK', [('Content-Type', 'application/xhtml+xml; charset=utf-8')])
return [template.render({'url': get_config().get('reports', 'urlRoot') + guid, 'knownIssues': knownIssues}).encode('utf-8')]
def showError(message, start_response):
template = get_template(get_config().get('reports', 'errorTemplate'))
start_response('400 Processing Error', [('Content-Type', 'application/xhtml+xml; charset=utf-8')])
return [template.render({'message': message}).encode('utf-8')]
def saveReport(guid, reportData, isNew=False):
cursor = get_db().cursor()
screenshot = reportData.get('screenshot', None)
if screenshot != None:
reportData['hasscreenshot'] = 2 if reportData.get(
'screenshotEdited', False) else 1
try:
saveScreenshot(guid, screenshot)
except (TypeError, UnicodeEncodeError):
reportData['hasscreenshot'] = 0
del reportData['screenshot']
knownIssues = len(reportData.get('knownIssues', []))
contact = getUserId(reportData.get('email', None)) if reportData.get(
'email', None) else None
dumpstr = marshal.dumps(reportData)
if contact != None and isNew:
executeQuery(
cursor,
'INSERT INTO #PFX#users (id, reports) VALUES (%s, 1) ON DUPLICATE KEY UPDATE reports = reports + 1',
contact)
executeQuery(
cursor,
'''INSERT INTO #PFX#reports (guid, type, ctime, site, comment, status, contact, hasscreenshot, knownissues, dump)
VALUES (%(guid)s, %(type)s, FROM_UNIXTIME(%(ctime)s), %(site)s, %(comment)s, %(status)s, %(contact)s,
%(hasscreenshot)s, %(knownissues)s, _binary %(dump)s) ON DUPLICATE KEY
UPDATE type = %(type)s, site = %(site)s, comment = %(comment)s, status = %(status)s,
hasscreenshot = %(hasscreenshot)s, knownissues = %(knownissues)s, dump = _binary %(dump)s''',
{
'guid': guid,
'type': reportData.get('type', None),
'ctime': reportData['time'],
'site': reportData.get('siteName', None),
'comment': reportData.get('comment', None),
'status': reportData.get('status', None),
'contact': contact,
'hasscreenshot': reportData.get('hasscreenshot', 0),
'knownissues': knownIssues,
'dump': dumpstr
})
if len(reportData['subscriptions']) > 0:
for sn in reportData['subscriptions']:
executeQuery(cursor,
'SELECT id FROM #PFX#subscriptions WHERE url = %s',
sn['id'])
id = cursor.fetchone()
if id != None:
def filterMatch(f):
return any(u == sn['id']
for u in f.get('subscriptions', []))
hasMatches = any(
filterMatch(f) for f in reportData.get('filters', []))
executeQuery(
cursor,
'INSERT IGNORE INTO #PFX#sublists (report, list, hasmatches) VALUES (%s, %s, %s)',
(guid, id[0], hasMatches))
get_db().commit()
reportData['guid'] = guid
if contact:
# TODO: The mail anonymization should happen in the template, not here
origEmail = reportData['email']
email = reportData['email']
email = re.sub(r' at ', r'@', email)
email = re.sub(r' dot ', r'.', email)
reportData['email'] = anonymizeMail(email)
reportData['uid'] = contact
file = os.path.join(get_config().get('reports', 'dataPath'), guid[0],
guid[1], guid[2], guid[3], guid + '.html')
dir = os.path.dirname(file)
if not os.path.exists(dir):
os.makedirs(dir)
template = get_template(get_config().get('reports', 'webTemplate'))
template.stream(reportData).dump(file, encoding='utf-8')
if contact:
reportData['email'] = origEmail
for subscription in subscriptions:
s = {'name': subscription.name, 'links': []}
result.append(s)
for key in ('homepage', 'forum', 'blog', 'faq', 'contact', 'changelog', 'policy'):
url = getattr(subscription, key)
if url != None:
site = urlparse(url).netloc
s['links'].append({
'url': url,
'title': key[0].upper() + key[1:],
'result': urls[url],
'siteResult': site in sites and sites[site],
})
for (title, url, complete) in subscription.variants:
site = urlparse(url).netloc
s['links'].append({
'url': url,
'title': title,
'result': urls[url],
'siteResult': site in sites and sites[site],
})
return result
if __name__ == '__main__':
setupStderr()
subscriptions = checkSubscriptions()
outputFile = get_config().get('subscriptions', 'statusPage')
template = get_template(get_config().get('subscriptions', 'statusTemplate'))
template.stream({'subscriptions': subscriptions}).dump(outputFile, encoding='utf-8')
url = getattr(subscription, key)
if url != None:
site = urlparse(url).netloc
s['links'].append({
'url': url,
'title': key[0].upper() + key[1:],
'result': urls[url],
'siteResult': site in sites and sites[site],
})
for (title, url, complete) in subscription.variants:
site = urlparse(url).netloc
s['links'].append({
'url': url,
'title': title,
'result': urls[url],
'siteResult': site in sites and sites[site],
})
return result
if __name__ == '__main__':
setupStderr()
subscriptions = checkSubscriptions()
outputFile = get_config().get('subscriptions', 'statusPage')
template = get_template(get_config().get('subscriptions',
'statusTemplate'))
template.stream({
'subscriptions': subscriptions
}).dump(outputFile, encoding='utf-8')
def updateDigests(dir):
global currentTime
subs = subscriptionParser.readSubscriptions()
defname, defemail = parseaddr(get_config().get(
'reports', 'defaultSubscriptionRecipient'))
subscriptions = {}
emails = {}
emails[defemail] = []
for subscription in subs.values():
for title, url, complete in subscription.variants:
subscriptions[url] = subscription
name, email = parseaddr(subscription.email)
if email != '':
emails[email] = []
startTime = currentTime - get_config().getint('reports',
'digestDays') * 24 * 60 * 60
for dbreport in getReports(startTime):
report = {
'guid':
dbreport['guid'],
'status':
dbreport['status'],
'url':
get_config().get('reports', 'urlRoot') + dbreport['guid'] +
'#secret=' + calculateReportSecret(dbreport['guid']),
'site':
dbreport['site'],
'comment':
dbreport['comment'],
'type':
dbreport['type'],
'subscriptions': [],
'contact':
dbreport['contact'],
'score':
getUserUsefulnessScore(dbreport['contact']),
'hasscreenshot':
dbreport['hasscreenshot'],
'knownIssues':
dbreport['knownissues'],
'time':
dbreport['ctime'],
}
recipients = set()
reportSubscriptions = getReportSubscriptions(dbreport['guid'])
if dbreport['type'] == 'false positive' or dbreport[
'type'] == 'false negative':
for subscription in reportSubscriptions:
subscriptionID = subscription.get('url', 'unknown')
# Send false negatives to all subscription authors, false positives
# only to subscriptions with matching filters
if subscriptionID in subscriptions and (
dbreport['type'] == 'false negative'
or subscription.get('hasmatches', 0) > 0):
name, email = parseaddr(
subscriptions[subscriptionID].email)
if email and not email in recipients:
recipients.add(email)
emails[email].append(report)
report['subscriptions'].append(
getSubscriptionInfo(subscriptions[subscriptionID]))
else:
for subscription in reportSubscriptions:
subscriptionID = subscription.get('url', 'unknown')
report['subscriptions'].append(
getSubscriptionInfo(subscriptions[subscriptionID]))
recipients.add(defemail)
emails[defemail].append(report)
# Generate new digests
digests = set()
for email, reports in emails.iteritems():
if len(reports) == 0:
continue
file = getDigestPath(dir, email)
template = get_template(get_config().get('reports',
'htmlDigestTemplate'))
template.stream({
'email': email,
'reports': reports
}).dump(file, encoding='utf-8')
digests.add(file)
# Remove not updated digests which are more then 2 weeks old
for filename in os.listdir(dir):
file = os.path.join(dir, filename)
if os.path.isfile(file) and file not in digests and re.match(
r'^[\da-f]{32}\.html$', filename
) and os.stat(file).st_mtime < currentTime - 14 * 24 * 60 * 60:
os.remove(file)
def updateDigests(dir):
global currentTime
subs = subscriptionParser.readSubscriptions()
defname, defemail = parseaddr(get_config().get('reports', 'defaultSubscriptionRecipient'))
subscriptions = {}
emails = {}
emails[defemail] = []
for subscription in subs.values():
for title, url, complete in subscription.variants:
subscriptions[url] = subscription
name, email = parseaddr(subscription.email)
if email != '':
emails[email] = []
startTime = currentTime - get_config().getint('reports', 'digestDays') * 24*60*60
for dbreport in getReports(startTime):
report = {
'guid': dbreport['guid'],
'status': dbreport['status'],
'url': get_config().get('reports', 'urlRoot') + dbreport['guid'] + '#secret=' + calculateReportSecret(dbreport['guid']),
'site': dbreport['site'],
'comment': dbreport['comment'],
'type': dbreport['type'],
'subscriptions': [],
'contact': dbreport['contact'],
'score': getUserUsefulnessScore(dbreport['contact']),
'hasscreenshot': dbreport['hasscreenshot'],
'knownIssues': dbreport['knownissues'],
'time': dbreport['ctime'],
}
recipients = set()
reportSubscriptions = getReportSubscriptions(dbreport['guid'])
if dbreport['type'] == 'false positive' or dbreport['type'] == 'false negative':
for subscription in reportSubscriptions:
subscriptionID = subscription.get('url', 'unknown')
# Send false negatives to all subscription authors, false positives
# only to subscriptions with matching filters
if subscriptionID in subscriptions and (dbreport['type'] == 'false negative' or subscription.get('hasmatches', 0) > 0):
name, email = parseaddr(subscriptions[subscriptionID].email)
if email and not email in recipients:
recipients.add(email)
emails[email].append(report)
report['subscriptions'].append(getSubscriptionInfo(subscriptions[subscriptionID]))
else:
for subscription in reportSubscriptions:
subscriptionID = subscription.get('url', 'unknown')
report['subscriptions'].append(getSubscriptionInfo(subscriptions[subscriptionID]))
recipients.add(defemail)
emails[defemail].append(report)
# Generate new digests
digests = set()
for email, reports in emails.iteritems():
if len(reports) == 0:
continue
file = getDigestPath(dir, email)
template = get_template(get_config().get('reports', 'htmlDigestTemplate'))
template.stream({'email': email, 'reports': reports}).dump(file, encoding='utf-8')
digests.add(file)
# Remove not updated digests which are more then 2 weeks old
for filename in os.listdir(dir):
file = os.path.join(dir, filename)
if os.path.isfile(file) and file not in digests and re.match(r'^[\da-f]{32}\.html$', filename) and os.stat(file).st_mtime < currentTime - 14*24*60*60:
os.remove(file)
def showError(message, start_response):
template = get_template(get_config().get("reports", "errorTemplate"))
start_response("400 Processing Error", [("Content-Type", "application/xhtml+xml; charset=utf-8")])
return [template.render({"message": message}).encode("utf-8")]
def showError(message, start_response):
template = get_template(get_config().get('reports', 'errorTemplate'))
start_response('400 Processing Error',
[('Content-Type', 'application/xhtml+xml; charset=utf-8')])
return [template.render({'message': message}).encode('utf-8')]
def test_get_template_default_path():
"""Load template from inside sitescripts."""
template = get_template('__init__.py')
assert template.render({}).startswith('# This file')
def showError(message, start_response):
template = get_template(get_config().get('reports', 'errorTemplate'))
start_response('400 Processing Error', [('Content-Type', 'application/xhtml+xml; charset=utf-8')])
return [template.render({'message': message}).encode('utf-8')]
def updateDigests(dir):
global currentTime
subs = subscriptionParser.readSubscriptions()
defname, defemail = parseaddr(get_config().get("reports", "defaultSubscriptionRecipient"))
subscriptions = {}
emails = {}
emails[defemail] = []
for subscription in subs.values():
for title, url, complete in subscription.variants:
subscriptions[url] = subscription
name, email = parseaddr(subscription.email)
if email != "":
emails[email] = []
startTime = currentTime - get_config().getint("reports", "digestDays") * 24 * 60 * 60
for dbreport in getReports(startTime):
report = {
"guid": dbreport["guid"],
"status": dbreport["status"],
"url": get_config().get("reports", "urlRoot")
+ dbreport["guid"]
+ "#secret="
+ calculateReportSecret(dbreport["guid"]),
"site": dbreport["site"],
"comment": dbreport["comment"],
"type": dbreport["type"],
"subscriptions": [],
"contact": dbreport["contact"],
"score": getUserUsefulnessScore(dbreport["contact"]),
"hasscreenshot": dbreport["hasscreenshot"],
"knownIssues": dbreport["knownissues"],
"time": dbreport["ctime"],
}
recipients = set()
reportSubscriptions = getReportSubscriptions(dbreport["guid"])
if dbreport["type"] == "false positive" or dbreport["type"] == "false negative":
for subscription in reportSubscriptions:
subscriptionID = subscription.get("url", "unknown")
# Send false negatives to all subscription authors, false positives
# only to subscriptions with matching filters
if subscriptionID in subscriptions and (
dbreport["type"] == "false negative" or subscription.get("hasmatches", 0) > 0
):
name, email = parseaddr(subscriptions[subscriptionID].email)
if email and not email in recipients:
recipients.add(email)
emails[email].append(report)
report["subscriptions"].append(getSubscriptionInfo(subscriptions[subscriptionID]))
else:
for subscription in reportSubscriptions:
subscriptionID = subscription.get("url", "unknown")
report["subscriptions"].append(getSubscriptionInfo(subscriptions[subscriptionID]))
recipients.add(defemail)
emails[defemail].append(report)
# Generate new digests
digests = set()
for email, reports in emails.iteritems():
if len(reports) == 0:
continue
file = getDigestPath(dir, email)
template = get_template(get_config().get("reports", "htmlDigestTemplate"))
template.stream({"email": email, "reports": reports}).dump(file, encoding="utf-8")
digests.add(file)
# Remove not updated digests which are more then 2 weeks old
for filename in os.listdir(dir):
file = os.path.join(dir, filename)
if (
os.path.isfile(file)
and file not in digests
and re.match(r"^[\da-f]{32}\.html$", filename)
and os.stat(file).st_mtime < currentTime - 14 * 24 * 60 * 60
):
os.remove(file)
