def test_get_template(tmpdir, mode): """Load template using relative or absolute path.""" template_path = tmpdir.join('template.tmpl') template_path.write('value = {{ value }}') if mode == 'absolute': template = get_template(template_path.strpath) else: template = get_template('template.tmpl', template_path=tmpdir.strpath) assert template.render({'value': 1}) == 'value = 1'
def make_handler(name, config): try: url = config['url'].value except (KeyError, AttributeError): raise Exception('No URL configured for form handler: ' + name) try: template = config['template'].value get_template(template, autoescape=False) except (KeyError, AttributeError): raise Exception('No template configured for form handler: ' + name) except jinja2.TemplateNotFound: raise Exception('Template not found at: ' + template) try: fields = config['fields'] for field, spec in fields.items(): spec.value = {s.strip() for s in spec.value.split(',')} except KeyError: raise Exception('No fields configured for form handler: ' + name) if len(fields) == 0: raise Exception('No fields configured for form handler: ' + name) @form_handler def handler(environ, start_response, params): response_headers = [('Content-Type', 'text/plain; charset=utf-8')] errors = validate_fields(fields, params) if errors: start_response('400 Bad Request', response_headers) return '\n'.join(errors) time = datetime.datetime.now() template_args = { 'time': time, 'fields': {field: params.get(field, '') for field in fields}, } try: sendMail(template, template_args) except: print(traceback.print_exc(), file=sys.stderr) start_response('500 Server Error', response_headers) return '' finally: if 'csv_log' in config: params = { field: params.get(field, '').encode('utf8') for field in fields } params['time'] = time log_formdata(params, config['csv_log'].value) start_response('200 OK', response_headers) return '' return url, handler
def writeUpdateManifest(self): """ Writes update manifest for the current build """ baseDir = os.path.join(self.config.nightliesDirectory, self.basename) if self.config.type == 'safari': manifestPath = os.path.join(baseDir, 'updates.plist') templateName = 'safariUpdateManifest' autoescape = True elif self.config.type == 'android': manifestPath = os.path.join(baseDir, 'updates.xml') templateName = 'androidUpdateManifest' autoescape = True else: return if not os.path.exists(baseDir): os.makedirs(baseDir) # ABP for Android used to have its own update manifest format. We need to # generate both that and the new one in the libadblockplus format as long # as a significant amount of users is on an old version. if self.config.type == 'android': newManifestPath = os.path.join(baseDir, 'update.json') writeAndroidUpdateManifest(newManifestPath, [{ 'basename': self.basename, 'version': self.version, 'updateURL': self.updateURL, }]) template = get_template(get_config().get('extensions', templateName), autoescape=autoescape) template.stream({'extensions': [self]}).dump(manifestPath)
def sitekey_frame(environ, start_response): template_path, template_file = os.path.split( get_config().get('testpages', 'sitekeyFrameTemplate'), ) template = get_template(template_file, template_path=template_path) http_path = request_path(environ) http_host = environ['HTTP_HOST'] http_ua = environ['HTTP_USER_AGENT'] key = M2Crypto.EVP.load_key(get_config().get('testpages', 'sitekeyPath')) key.sign_init() key.sign_update('\x00'.join([http_path, http_host, http_ua])) public_key = base64.b64encode(key.as_der()) signature = base64.b64encode(key.final()) start_response('200 OK', [ ('Content-Type', 'text/html; charset=utf-8'), ('X-Adblock-Key', '%s_%s' % (public_key, signature)), ]) return [template.render({ 'public_key': public_key, 'signature': signature, 'http_path': http_path, 'http_host': http_host, 'http_ua': http_ua, }).encode('utf-8')]
def updateIndex(self, versions): """ Updates index page listing all existing versions """ baseDir = os.path.join(self.config.nightliesDirectory, self.basename) if not os.path.exists(baseDir): os.makedirs(baseDir) outputFile = 'index.html' outputPath = os.path.join(baseDir, outputFile) links = [] for version in versions: packageFile = self.basename + '-' + version + self.config.packageSuffix changelogFile = self.basename + '-' + version + '.changelog.xhtml' if not os.path.exists(os.path.join(baseDir, packageFile)): # Oops continue link = { 'version': version, 'download': packageFile, 'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)), 'size': os.path.getsize(os.path.join(baseDir, packageFile)), } if os.path.exists(os.path.join(baseDir, changelogFile)): link['changelog'] = changelogFile links.append(link) template = get_template(get_config().get('extensions', 'nightlyIndexPage')) template.stream({'config': self.config, 'links': links}).dump(outputPath)
def write(self): template = get_template(self.repo.padTemplate) basename = self.repo.basename filename = basename + '.xml' pad = template.render({ 'name': self.repo.name, 'type': self.repo.type, 'basename': basename, 'browser_name': self.browser_name, 'browser_min_version': self.browser_min_version, 'version': self.version, 'release_date': self.release_date, 'release_status': self.release_status, 'os_support': ','.join(self.os_support), 'language': ','.join(self.languages), 'download_size': self.download_size, 'download_url': self.download_url, 'pad_url': urljoin(self.repo.padURL, filename), }).encode('utf-8') path = os.path.join(self.repo.padDirectory, filename) validate_pad(pad, path) with open(path, 'wb') as file: file.write(pad)
def writeUpdateManifest(self): """ Writes update.rdf file for the current build """ baseDir = os.path.join(self.config.nightliesDirectory, self.basename) if not os.path.exists(baseDir): os.makedirs(baseDir) if self.config.type == 'chrome' or self.config.type == 'opera': manifestPath = os.path.join(baseDir, "updates.xml") templateName = 'chromeUpdateManifest' elif self.config.type == 'safari': manifestPath = os.path.join(baseDir, "updates.plist") templateName = 'safariUpdateManifest' elif self.config.type == 'android': manifestPath = os.path.join(baseDir, "updates.xml") templateName = 'androidUpdateManifest' # ABP for Android used to have its own update manifest format. We need to # generate both that and the new one in the libadblockplus format as long # as a significant amount of users is on an old version. newManifestPath = os.path.join(baseDir, "update.json") writeAndroidUpdateManifest(newManifestPath, [{ 'basename': self.basename, 'version': self.version, 'updateURL': self.updateURL }]) else: manifestPath = os.path.join(baseDir, "update.rdf") templateName = 'geckoUpdateManifest' template = get_template(get_config().get('extensions', templateName)) template.stream({'extensions': [self]}).dump(manifestPath)
def writeUpdateManifest(links): """ writes an update manifest for all extensions and Android apps """ extensions = {'gecko': [], 'android': [], 'safari': [], 'ie': []} for repo in Configuration.getRepositoryConfigurations(): if repo.type not in extensions or not links.has_section(repo.repositoryName): continue data = readMetadata(repo, links.get(repo.repositoryName, 'version')) data['updateURL'] = links.get(repo.repositoryName, 'downloadURL') if data['updateURL'].startswith(repo.downloadsURL): data['updateURL'] += "?update" extensions[repo.type].append(data) if len(extensions['android']) > 1: print >>sys.stderr, 'Warning: more than one Android app defined, update manifest only works for one' for repoType in extensions.iterkeys(): manifestPath = get_config().get('extensions', '%sUpdateManifestPath' % repoType) if repoType == 'ie': writeIEUpdateManifest(manifestPath, extensions[repoType]) else: # ABP for Android used to have its own update manifest format. We need to # generate both that and the new one in the libadblockplus format as long # as a significant amount of users is on an old version. if repoType == 'android': newManifestPath = get_config().get("extensions", "androidNewUpdateManifestPath") writeAndroidUpdateManifest(newManifestPath, extensions[repoType]) template = get_template(get_config().get('extensions', '%sUpdateManifest' % repoType)) template.stream({'extensions': extensions[repoType]}).dump(manifestPath)
def sitekey_frame(environ, start_response): template_path, template_file = os.path.split( get_config().get('testpages', 'sitekeyFrameTemplate'), ) template = get_template(template_file, template_path=template_path) key = M2Crypto.EVP.load_key(get_config().get('testpages', 'sitekeyPath')) key.sign_init() key.sign_update('\x00'.join(( request_path(environ), environ['HTTP_HOST'], environ['HTTP_USER_AGENT'], ))) public_key = base64.b64encode(key.as_der()) signature = base64.b64encode(key.final()) start_response('200 OK', [('Content-Type', 'text/html; charset=utf-8'), ('X-Adblock-Key', '%s_%s' % (public_key, signature))]) return [ template.render({ 'public_key': public_key, 'signature': signature }).encode('utf-8') ]
def handleRequest(environ, start_response): if not environ.get('HTTP_X_ADBLOCK_PLUS'): return showError('Please use Adblock Plus to submit reports', start_response) if environ['REQUEST_METHOD'].upper() != 'POST' or not environ.get( 'CONTENT_TYPE', '').startswith('text/xml'): return showError('Unsupported request method', start_response) params = parse_qs(environ.get('QUERY_STRING', '')) requestVersion = params.get('version', ['0'])[0] if requestVersion != '1': return showError('Unsupported request version', start_response) guid = params.get('guid', [''])[0].lower() if not re.match( r'^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$', guid): return showError('Invalid or missing GUID', start_response) path = os.path.join(get_config().get('reports', 'dataPath'), guid + '.xml') if os.path.exists(path) or os.path.exists(path + '.tmp'): return showError('Duplicate GUID', start_response) try: request_size = int(environ['CONTENT_LENGTH']) except (KeyError, ValueError): return showError('Invalid or missing Content-Length header', start_response, '411 Length Required') dir = os.path.dirname(path) if not os.path.exists(dir): os.makedirs(dir) try: file = open(path + '.tmp', 'wb') data = environ['wsgi.input'].read(request_size) file.write(data) file.close() knownIssues = knownIssuesParser.findMatches( data.splitlines(), params.get('lang', ['en-US'])[0]) os.rename(path + '.tmp', path) except Exception as e: if os.path.isfile(path + '.tmp'): os.remove(path + '.tmp') raise e template = get_template(get_config().get('reports', 'submitResponseTemplate')) start_response('200 OK', [('Content-Type', 'application/xhtml+xml; charset=utf-8')]) return [ template.render({ 'url': get_config().get('reports', 'urlRoot') + guid, 'knownIssues': knownIssues }).encode('utf-8') ]
def writeUpdateManifest(self): """ Writes update.rdf file for the current build """ baseDir = os.path.join(self.config.nightliesDirectory, self.basename) if self.config.type == 'safari': manifestPath = os.path.join(baseDir, 'updates.plist') templateName = 'safariUpdateManifest' elif self.config.type == 'android': manifestPath = os.path.join(baseDir, 'updates.xml') templateName = 'androidUpdateManifest' else: return if not os.path.exists(baseDir): os.makedirs(baseDir) # ABP for Android used to have its own update manifest format. We need to # generate both that and the new one in the libadblockplus format as long # as a significant amount of users is on an old version. if self.config.type == 'android': newManifestPath = os.path.join(baseDir, 'update.json') writeAndroidUpdateManifest(newManifestPath, [{ 'basename': self.basename, 'version': self.version, 'updateURL': self.updateURL }]) template = get_template(get_config().get('extensions', templateName)) template.stream({'extensions': [self]}).dump(manifestPath)
def updateIndex(self, versions): """ Updates index page listing all existing versions """ baseDir = os.path.join(self.config.nightliesDirectory, self.basename) if not os.path.exists(baseDir): os.makedirs(baseDir) outputFile = "index.html" outputPath = os.path.join(baseDir, outputFile) links = [] for version in versions: packageFile = self.basename + '-' + version + self.config.packageSuffix changelogFile = self.basename + '-' + version + '.changelog.xhtml' if not os.path.exists(os.path.join(baseDir, packageFile)): # Oops continue link = { 'version': version, 'download': packageFile, 'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)), 'size': os.path.getsize(os.path.join(baseDir, packageFile)) } if os.path.exists(os.path.join(baseDir, changelogFile)): link['changelog'] = changelogFile links.append(link) template = get_template(get_config().get('extensions', 'nightlyIndexPage')) template.stream({'config': self.config, 'links': links}).dump(outputPath)
def writeChangelog(self, changes): """ write the changelog file into the cloned repository """ baseDir = os.path.join(self.config.nightliesDirectory, self.basename) if not os.path.exists(baseDir): os.makedirs(baseDir) changelogFile = "%s-%s.changelog.xhtml" % (self.basename, self.version) changelogPath = os.path.join(baseDir, changelogFile) self.changelogURL = urlparse.urljoin( self.config.nightliesURL, self.basename + '/' + changelogFile) template = get_template(get_config().get('extensions', 'changelogTemplate')) template.stream({ 'changes': changes }).dump(changelogPath, encoding='utf-8') linkPath = os.path.join(baseDir, '00latest.changelog.xhtml') if hasattr(os, 'symlink'): if os.path.exists(linkPath): os.remove(linkPath) os.symlink(os.path.basename(changelogPath), linkPath) else: shutil.copyfile(changelogPath, linkPath)
def _generate_comments(repository_name, changes_by_issue): comments = {} template = get_template("hg/template/issue_commit_comment.tmpl", autoescape=False) for issue_id, changes in changes_by_issue.iteritems(): comments[issue_id] = template.render({"repository_name": repository_name, "changes": changes}) return comments
def writeSubscriptions(templateName, outputFile=None): subscriptions = subscriptionParser.readSubscriptions().values() template = get_template(get_config().get('subscriptions', templateName + 'Template')) if outputFile == None: outputFile = get_config().get('subscriptions', templateName + 'File') template.stream({ 'subscriptions': subscriptions }).dump(outputFile, encoding='utf-8')
def saveReport(guid, reportData, isNew=False): cursor = get_db().cursor() screenshot = reportData.get('screenshot', None) if screenshot != None: reportData['hasscreenshot'] = 2 if reportData.get('screenshotEdited', False) else 1 try: saveScreenshot(guid, screenshot) except (TypeError, UnicodeEncodeError): reportData['hasscreenshot'] = 0 del reportData['screenshot'] knownIssues = len(reportData.get('knownIssues', [])) contact = getUserId(reportData.get('email', None)) if reportData.get('email', None) else None dumpstr = marshal.dumps(reportData) if contact != None and isNew: executeQuery(cursor, 'INSERT INTO #PFX#users (id, reports) VALUES (%s, 1) ON DUPLICATE KEY UPDATE reports = reports + 1', contact) executeQuery(cursor, '''INSERT INTO #PFX#reports (guid, type, ctime, site, comment, status, contact, hasscreenshot, knownissues, dump) VALUES (%(guid)s, %(type)s, FROM_UNIXTIME(%(ctime)s), %(site)s, %(comment)s, %(status)s, %(contact)s, %(hasscreenshot)s, %(knownissues)s, _binary %(dump)s) ON DUPLICATE KEY UPDATE type = %(type)s, site = %(site)s, comment = %(comment)s, status = %(status)s, hasscreenshot = %(hasscreenshot)s, knownissues = %(knownissues)s, dump = _binary %(dump)s''', {'guid': guid, 'type': reportData.get('type', None), 'ctime': reportData['time'], 'site': reportData.get('siteName', None), 'comment': reportData.get('comment', None), 'status': reportData.get('status', None), 'contact': contact, 'hasscreenshot': reportData.get('hasscreenshot', 0), 'knownissues': knownIssues, 'dump': dumpstr}) if len(reportData['subscriptions']) > 0: for sn in reportData['subscriptions']: executeQuery(cursor, 'SELECT id FROM #PFX#subscriptions WHERE url = %s', sn['id']) id = cursor.fetchone() if id != None: def filterMatch(f): return any(u == sn['id'] for u in f.get('subscriptions', [])) hasMatches = any(filterMatch(f) for f in reportData.get('filters', [])) executeQuery(cursor, 'INSERT IGNORE INTO #PFX#sublists (report, list, hasmatches) VALUES (%s, %s, %s)', (guid, id[0], hasMatches)) get_db().commit() reportData['guid'] = guid if contact: # TODO: The mail anonymization should happen in the template, not here origEmail = reportData['email'] email = reportData['email'] email = re.sub(r' at ', r'@', email) email = re.sub(r' dot ', r'.', email) reportData['email'] = anonymizeMail(email) reportData['uid'] = contact file = os.path.join(get_config().get('reports', 'dataPath'), guid[0], guid[1], guid[2], guid[3], guid + '.html') dir = os.path.dirname(file) if not os.path.exists(dir): os.makedirs(dir) template = get_template(get_config().get('reports', 'webTemplate')) template.stream(reportData).dump(file, encoding='utf-8') if contact: reportData['email'] = origEmail
def _post_comments(ui, repo, config, refs): repo_name = posixpath.split(repo.url())[1] template = get_template('hg/template/issue_commit_comment.tmpl', autoescape=False) for ref in refs: comment_text = template.render({ 'repository_name': repo_name, 'changes': ref.commits, 'format_description': _format_description, }) with _trac_proxy(ui, config, 'getting issue {}'.format(ref.id)) as tp: attrs = tp.ticket.get(ref.id)[3] changes = {'_ts': attrs['_ts'], 'action': 'leave'} _update_issue(ui, config, ref.id, changes, comment_text)
def _generate_comments(repository_name, changes_by_issue): comments = {} template = get_template('hg/template/issue_commit_comment.tmpl', autoescape=False) for issue_id, changes in changes_by_issue.items(): comments[issue_id] = template.render({ 'repository_name': repository_name, 'changes': changes, 'format_description': _format_description, }) return comments
def sitekey_frame(environ, start_response): template_path, template_file = os.path.split(get_config().get("testpages", "sitekeyFrameTemplate")) template = get_template(template_file, template_path=template_path) key = M2Crypto.EVP.load_key(get_config().get("testpages", "sitekeyPath")) key.sign_init() key.sign_update("\x00".join((request_path(environ), environ["HTTP_HOST"], environ["HTTP_USER_AGENT"]))) public_key = base64.b64encode(key.as_der()) signature = base64.b64encode(key.final()) start_response( "200 OK", [("Content-Type", "text/html; charset=utf-8"), ("X-Adblock-Key", "%s_%s" % (public_key, signature))] ) return [template.render({"public_key": public_key, "signature": signature}).encode("utf-8")]
def handleRequest(environ, start_response): if not environ.get('HTTP_X_ADBLOCK_PLUS'): return showError('Please use Adblock Plus to submit reports', start_response) if environ['REQUEST_METHOD'].upper() != 'POST' or not environ.get('CONTENT_TYPE', '').startswith('text/xml'): return showError('Unsupported request method', start_response) params = parse_qs(environ.get('QUERY_STRING', '')) requestVersion = params.get('version', ['0'])[0] if requestVersion != '1': return showError('Unsupported request version', start_response) guid = params.get('guid', [''])[0].lower() if not re.match(r'^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$', guid): return showError('Invalid or missing GUID', start_response) path = os.path.join(get_config().get('reports', 'dataPath'), guid + '.xml') if os.path.exists(path) or os.path.exists(path + '.tmp'): return showError('Duplicate GUID', start_response) try: request_size = int(environ['CONTENT_LENGTH']) except (KeyError, ValueError): return showError('Invalid or missing Content-Length header', start_response, '411 Length Required') dir = os.path.dirname(path) if not os.path.exists(dir): os.makedirs(dir) try: file = open(path + '.tmp', 'wb') data = environ['wsgi.input'].read(request_size) file.write(data) file.close() knownIssues = knownIssuesParser.findMatches(data.splitlines(), params.get('lang', ['en-US'])[0]) os.rename(path + '.tmp', path) except Exception as e: if os.path.isfile(path + '.tmp'): os.remove(path + '.tmp') raise e template = get_template(get_config().get('reports', 'submitResponseTemplate')) start_response('200 OK', [('Content-Type', 'application/xhtml+xml; charset=utf-8')]) return [template.render({'url': get_config().get('reports', 'urlRoot') + guid, 'knownIssues': knownIssues}).encode('utf-8')]
def handleRequest(environ, start_response): setupStderr(environ['wsgi.errors']) params = parse_qs(environ.get('QUERY_STRING', '')) id = params.get('id', [''])[0].lower() if not re.match(r'^[\da-f]{32}$', id): return showError('Invalid or missing ID', start_response) user = getUser(id) if user == None: return showError('User not found', start_response) user['reportlist'] = getReportsForUser(id) template = get_template(get_config().get('reports', 'showUserTemplate')) start_response('200 OK', [('Content-Type', 'application/xhtml+xml; charset=utf-8')]) return [template.render(user).encode('utf-8')]
def writeChangelog(self, changes): """ write the changelog file into the cloned repository """ baseDir = os.path.join(self.config.nightliesDirectory, self.basename) if not os.path.exists(baseDir): os.makedirs(baseDir) changelogFile = "%s-%s.changelog.xhtml" % (self.basename, self.version) changelogPath = os.path.join(baseDir, changelogFile) self.changelogURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '/' + changelogFile) template = get_template(get_config().get('extensions', 'changelogTemplate')) template.stream({'changes': changes}).dump(changelogPath) linkPath = os.path.join(baseDir, '00latest.changelog.xhtml') if hasattr(os, 'symlink'): if os.path.exists(linkPath): os.remove(linkPath) os.symlink(os.path.basename(changelogPath), linkPath) else: shutil.copyfile(changelogPath, linkPath)
def writeUpdateManifest(links): """ writes an update manifest for all extensions and Android apps """ extensions = {'android': [], 'safari': [], 'ie': []} for repo in Configuration.getRepositoryConfigurations(): if repo.type not in extensions or not links.has_section( repo.repositoryName): continue data = readMetadata(repo, links.get(repo.repositoryName, 'version')) data['updateURL'] = links.get(repo.repositoryName, 'downloadURL') if data['updateURL'].startswith(repo.downloadsURL): data['updateURL'] += '?update' extensions[repo.type].append(data) if len(extensions['android']) > 1: print >> sys.stderr, 'Warning: more than one Android app defined, update manifest only works for one' for repoType in extensions.iterkeys(): manifestPath = get_config().get('extensions', '%sUpdateManifestPath' % repoType) if repoType == 'ie': writeIEUpdateManifest(manifestPath, extensions[repoType]) else: # ABP for Android used to have its own update manifest format. We need to # generate both that and the new one in the libadblockplus format as long # as a significant amount of users is on an old version. if repoType == 'android': newManifestPath = get_config().get( 'extensions', 'androidNewUpdateManifestPath') writeAndroidUpdateManifest(newManifestPath, extensions[repoType]) path = get_config().get('extensions', '%sUpdateManifest' % repoType) template = get_template(path, autoescape=not path.endswith('.json')) template.stream({ 'extensions': extensions[repoType] }).dump(manifestPath)
def writeUpdateManifest(links): """ writes an update manifest for all Gecko extensions and Android apps """ extensions = {'gecko': [], 'android': []} for repo in Configuration.getRepositoryConfigurations(): if repo.type not in extensions or not links.has_section(repo.repositoryName): continue data = readMetadata(repo, links.get(repo.repositoryName, 'version')) data['updateURL'] = links.get(repo.repositoryName, 'downloadURL') if data['updateURL'].startswith(repo.downloadsURL): data['updateURL'] += "?update" extensions[repo.type].append(data) if len(extensions['android']) > 1: print >>sys.stderr, 'Warning: more than one Android app defined, update manifest only works for one' for repoType in extensions.iterkeys(): manifestPath = get_config().get('extensions', '%sUpdateManifestPath' % repoType) template = get_template(get_config().get('extensions', '%sUpdateManifest' % repoType)) template.stream({'extensions': extensions[repoType]}).dump(manifestPath)
def writeUpdateManifest(self): """ Writes update.rdf file for the current build """ baseDir = os.path.join(self.config.nightliesDirectory, self.basename) if not os.path.exists(baseDir): os.makedirs(baseDir) if self.config.type == 'chrome' or self.config.type == 'opera': manifestPath = os.path.join(baseDir, "updates.xml") templateName = 'chromeUpdateManifest' elif self.config.type == 'safari': manifestPath = os.path.join(baseDir, "updates.plist") templateName = 'safariUpdateManifest' elif self.config.type == 'android': manifestPath = os.path.join(baseDir, "updates.xml") templateName = 'androidUpdateManifest' else: manifestPath = os.path.join(baseDir, "update.rdf") templateName = 'geckoUpdateManifest' template = get_template(get_config().get('extensions', templateName)) template.stream({'extensions': [self]}).dump(manifestPath)
def sitekey_frame(environ, start_response): template_path, template_file = os.path.split(get_config().get( "testpages", "sitekeyFrameTemplate")) template = get_template(template_file, template_path=template_path) key = M2Crypto.EVP.load_key(get_config().get("testpages", "sitekeyPath")) key.sign_init() key.sign_update("\0".join((request_path(environ), environ["HTTP_HOST"], environ["HTTP_USER_AGENT"]))) public_key = base64.b64encode(key.as_der()) signature = base64.b64encode(key.final()) start_response("200 OK", [("Content-Type", "text/html; charset=utf-8"), ("X-Adblock-Key", "%s_%s" % (public_key, signature))]) return [ template.render({ "public_key": public_key, "signature": signature }).encode("utf-8") ]
request_size= int(environ['CONTENT_LENGTH']) except (KeyError, ValueError): return showError('Invalid or missing Content-Length header', start_response, '411 Length Required') dir = os.path.dirname(path) if not os.path.exists(dir): os.makedirs(dir) try: file = open(path + '.tmp', 'wb') data = environ['wsgi.input'].read(request_size) file.write(data) file.close() knownIssues = knownIssuesParser.findMatches(data.splitlines(), params.get('lang', ['en-US'])[0]) os.rename(path + '.tmp', path); except Exception, e: if os.path.isfile(path + '.tmp'): os.remove(path + '.tmp') raise e template = get_template(get_config().get('reports', 'submitResponseTemplate')) start_response('200 OK', [('Content-Type', 'application/xhtml+xml; charset=utf-8')]) return [template.render({'url': get_config().get('reports', 'urlRoot') + guid, 'knownIssues': knownIssues}).encode('utf-8')] def showError(message, start_response, response_code='400 Processing Error'): template = get_template(get_config().get('reports', 'errorTemplate')) start_response(response_code, [('Content-Type', 'application/xhtml+xml; charset=utf-8')]) return [template.render({'message': message}).encode('utf-8')]
def writeSubscriptions(templateName, outputFile=None): subscriptions = subscriptionParser.readSubscriptions().values() template = get_template(get_config().get('subscriptions', templateName + 'Template')) if outputFile == None: outputFile = get_config().get('subscriptions', templateName + 'File') template.stream({'subscriptions': subscriptions}).dump(outputFile, encoding='utf-8')
if not re.match(r'^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$', guid): return showError('Invalid or missing GUID', start_response) path = os.path.join(get_config().get('reports', 'dataPath'), guid + '.xml') if os.path.exists(path) or os.path.exists(path + '.tmp'): return showError('Duplicate GUID', start_response) dir = os.path.dirname(path) if not os.path.exists(dir): os.makedirs(dir) try: file = open(path + '.tmp', 'wb') iter = dataIterator(environ['wsgi.input'], file) knownIssues = knownIssuesParser.findMatches(iter, params.get('lang', ['en-US'])[0]) file.close() os.rename(path + '.tmp', path); except Exception, e: if os.path.isfile(path + '.tmp'): os.remove(path + '.tmp') raise e template = get_template(get_config().get('reports', 'submitResponseTemplate')) start_response('200 OK', [('Content-Type', 'application/xhtml+xml; charset=utf-8')]) return [template.render({'url': get_config().get('reports', 'urlRoot') + guid, 'knownIssues': knownIssues}).encode('utf-8')] def showError(message, start_response): template = get_template(get_config().get('reports', 'errorTemplate')) start_response('400 Processing Error', [('Content-Type', 'application/xhtml+xml; charset=utf-8')]) return [template.render({'message': message}).encode('utf-8')]
def saveReport(guid, reportData, isNew=False): cursor = get_db().cursor() screenshot = reportData.get('screenshot', None) if screenshot != None: reportData['hasscreenshot'] = 2 if reportData.get( 'screenshotEdited', False) else 1 try: saveScreenshot(guid, screenshot) except (TypeError, UnicodeEncodeError): reportData['hasscreenshot'] = 0 del reportData['screenshot'] knownIssues = len(reportData.get('knownIssues', [])) contact = getUserId(reportData.get('email', None)) if reportData.get( 'email', None) else None dumpstr = marshal.dumps(reportData) if contact != None and isNew: executeQuery( cursor, 'INSERT INTO #PFX#users (id, reports) VALUES (%s, 1) ON DUPLICATE KEY UPDATE reports = reports + 1', contact) executeQuery( cursor, '''INSERT INTO #PFX#reports (guid, type, ctime, site, comment, status, contact, hasscreenshot, knownissues, dump) VALUES (%(guid)s, %(type)s, FROM_UNIXTIME(%(ctime)s), %(site)s, %(comment)s, %(status)s, %(contact)s, %(hasscreenshot)s, %(knownissues)s, _binary %(dump)s) ON DUPLICATE KEY UPDATE type = %(type)s, site = %(site)s, comment = %(comment)s, status = %(status)s, hasscreenshot = %(hasscreenshot)s, knownissues = %(knownissues)s, dump = _binary %(dump)s''', { 'guid': guid, 'type': reportData.get('type', None), 'ctime': reportData['time'], 'site': reportData.get('siteName', None), 'comment': reportData.get('comment', None), 'status': reportData.get('status', None), 'contact': contact, 'hasscreenshot': reportData.get('hasscreenshot', 0), 'knownissues': knownIssues, 'dump': dumpstr }) if len(reportData['subscriptions']) > 0: for sn in reportData['subscriptions']: executeQuery(cursor, 'SELECT id FROM #PFX#subscriptions WHERE url = %s', sn['id']) id = cursor.fetchone() if id != None: def filterMatch(f): return any(u == sn['id'] for u in f.get('subscriptions', [])) hasMatches = any( filterMatch(f) for f in reportData.get('filters', [])) executeQuery( cursor, 'INSERT IGNORE INTO #PFX#sublists (report, list, hasmatches) VALUES (%s, %s, %s)', (guid, id[0], hasMatches)) get_db().commit() reportData['guid'] = guid if contact: # TODO: The mail anonymization should happen in the template, not here origEmail = reportData['email'] email = reportData['email'] email = re.sub(r' at ', r'@', email) email = re.sub(r' dot ', r'.', email) reportData['email'] = anonymizeMail(email) reportData['uid'] = contact file = os.path.join(get_config().get('reports', 'dataPath'), guid[0], guid[1], guid[2], guid[3], guid + '.html') dir = os.path.dirname(file) if not os.path.exists(dir): os.makedirs(dir) template = get_template(get_config().get('reports', 'webTemplate')) template.stream(reportData).dump(file, encoding='utf-8') if contact: reportData['email'] = origEmail
for subscription in subscriptions: s = {'name': subscription.name, 'links': []} result.append(s) for key in ('homepage', 'forum', 'blog', 'faq', 'contact', 'changelog', 'policy'): url = getattr(subscription, key) if url != None: site = urlparse(url).netloc s['links'].append({ 'url': url, 'title': key[0].upper() + key[1:], 'result': urls[url], 'siteResult': site in sites and sites[site], }) for (title, url, complete) in subscription.variants: site = urlparse(url).netloc s['links'].append({ 'url': url, 'title': title, 'result': urls[url], 'siteResult': site in sites and sites[site], }) return result if __name__ == '__main__': setupStderr() subscriptions = checkSubscriptions() outputFile = get_config().get('subscriptions', 'statusPage') template = get_template(get_config().get('subscriptions', 'statusTemplate')) template.stream({'subscriptions': subscriptions}).dump(outputFile, encoding='utf-8')
url = getattr(subscription, key) if url != None: site = urlparse(url).netloc s['links'].append({ 'url': url, 'title': key[0].upper() + key[1:], 'result': urls[url], 'siteResult': site in sites and sites[site], }) for (title, url, complete) in subscription.variants: site = urlparse(url).netloc s['links'].append({ 'url': url, 'title': title, 'result': urls[url], 'siteResult': site in sites and sites[site], }) return result if __name__ == '__main__': setupStderr() subscriptions = checkSubscriptions() outputFile = get_config().get('subscriptions', 'statusPage') template = get_template(get_config().get('subscriptions', 'statusTemplate')) template.stream({ 'subscriptions': subscriptions }).dump(outputFile, encoding='utf-8')
def updateDigests(dir): global currentTime subs = subscriptionParser.readSubscriptions() defname, defemail = parseaddr(get_config().get( 'reports', 'defaultSubscriptionRecipient')) subscriptions = {} emails = {} emails[defemail] = [] for subscription in subs.values(): for title, url, complete in subscription.variants: subscriptions[url] = subscription name, email = parseaddr(subscription.email) if email != '': emails[email] = [] startTime = currentTime - get_config().getint('reports', 'digestDays') * 24 * 60 * 60 for dbreport in getReports(startTime): report = { 'guid': dbreport['guid'], 'status': dbreport['status'], 'url': get_config().get('reports', 'urlRoot') + dbreport['guid'] + '#secret=' + calculateReportSecret(dbreport['guid']), 'site': dbreport['site'], 'comment': dbreport['comment'], 'type': dbreport['type'], 'subscriptions': [], 'contact': dbreport['contact'], 'score': getUserUsefulnessScore(dbreport['contact']), 'hasscreenshot': dbreport['hasscreenshot'], 'knownIssues': dbreport['knownissues'], 'time': dbreport['ctime'], } recipients = set() reportSubscriptions = getReportSubscriptions(dbreport['guid']) if dbreport['type'] == 'false positive' or dbreport[ 'type'] == 'false negative': for subscription in reportSubscriptions: subscriptionID = subscription.get('url', 'unknown') # Send false negatives to all subscription authors, false positives # only to subscriptions with matching filters if subscriptionID in subscriptions and ( dbreport['type'] == 'false negative' or subscription.get('hasmatches', 0) > 0): name, email = parseaddr( subscriptions[subscriptionID].email) if email and not email in recipients: recipients.add(email) emails[email].append(report) report['subscriptions'].append( getSubscriptionInfo(subscriptions[subscriptionID])) else: for subscription in reportSubscriptions: subscriptionID = subscription.get('url', 'unknown') report['subscriptions'].append( getSubscriptionInfo(subscriptions[subscriptionID])) recipients.add(defemail) emails[defemail].append(report) # Generate new digests digests = set() for email, reports in emails.iteritems(): if len(reports) == 0: continue file = getDigestPath(dir, email) template = get_template(get_config().get('reports', 'htmlDigestTemplate')) template.stream({ 'email': email, 'reports': reports }).dump(file, encoding='utf-8') digests.add(file) # Remove not updated digests which are more then 2 weeks old for filename in os.listdir(dir): file = os.path.join(dir, filename) if os.path.isfile(file) and file not in digests and re.match( r'^[\da-f]{32}\.html$', filename ) and os.stat(file).st_mtime < currentTime - 14 * 24 * 60 * 60: os.remove(file)
def updateDigests(dir): global currentTime subs = subscriptionParser.readSubscriptions() defname, defemail = parseaddr(get_config().get('reports', 'defaultSubscriptionRecipient')) subscriptions = {} emails = {} emails[defemail] = [] for subscription in subs.values(): for title, url, complete in subscription.variants: subscriptions[url] = subscription name, email = parseaddr(subscription.email) if email != '': emails[email] = [] startTime = currentTime - get_config().getint('reports', 'digestDays') * 24*60*60 for dbreport in getReports(startTime): report = { 'guid': dbreport['guid'], 'status': dbreport['status'], 'url': get_config().get('reports', 'urlRoot') + dbreport['guid'] + '#secret=' + calculateReportSecret(dbreport['guid']), 'site': dbreport['site'], 'comment': dbreport['comment'], 'type': dbreport['type'], 'subscriptions': [], 'contact': dbreport['contact'], 'score': getUserUsefulnessScore(dbreport['contact']), 'hasscreenshot': dbreport['hasscreenshot'], 'knownIssues': dbreport['knownissues'], 'time': dbreport['ctime'], } recipients = set() reportSubscriptions = getReportSubscriptions(dbreport['guid']) if dbreport['type'] == 'false positive' or dbreport['type'] == 'false negative': for subscription in reportSubscriptions: subscriptionID = subscription.get('url', 'unknown') # Send false negatives to all subscription authors, false positives # only to subscriptions with matching filters if subscriptionID in subscriptions and (dbreport['type'] == 'false negative' or subscription.get('hasmatches', 0) > 0): name, email = parseaddr(subscriptions[subscriptionID].email) if email and not email in recipients: recipients.add(email) emails[email].append(report) report['subscriptions'].append(getSubscriptionInfo(subscriptions[subscriptionID])) else: for subscription in reportSubscriptions: subscriptionID = subscription.get('url', 'unknown') report['subscriptions'].append(getSubscriptionInfo(subscriptions[subscriptionID])) recipients.add(defemail) emails[defemail].append(report) # Generate new digests digests = set() for email, reports in emails.iteritems(): if len(reports) == 0: continue file = getDigestPath(dir, email) template = get_template(get_config().get('reports', 'htmlDigestTemplate')) template.stream({'email': email, 'reports': reports}).dump(file, encoding='utf-8') digests.add(file) # Remove not updated digests which are more then 2 weeks old for filename in os.listdir(dir): file = os.path.join(dir, filename) if os.path.isfile(file) and file not in digests and re.match(r'^[\da-f]{32}\.html$', filename) and os.stat(file).st_mtime < currentTime - 14*24*60*60: os.remove(file)
def showError(message, start_response): template = get_template(get_config().get("reports", "errorTemplate")) start_response("400 Processing Error", [("Content-Type", "application/xhtml+xml; charset=utf-8")]) return [template.render({"message": message}).encode("utf-8")]
def showError(message, start_response): template = get_template(get_config().get('reports', 'errorTemplate')) start_response('400 Processing Error', [('Content-Type', 'application/xhtml+xml; charset=utf-8')]) return [template.render({'message': message}).encode('utf-8')]
def test_get_template_default_path(): """Load template from inside sitescripts.""" template = get_template('__init__.py') assert template.render({}).startswith('# This file')
def updateDigests(dir): global currentTime subs = subscriptionParser.readSubscriptions() defname, defemail = parseaddr(get_config().get("reports", "defaultSubscriptionRecipient")) subscriptions = {} emails = {} emails[defemail] = [] for subscription in subs.values(): for title, url, complete in subscription.variants: subscriptions[url] = subscription name, email = parseaddr(subscription.email) if email != "": emails[email] = [] startTime = currentTime - get_config().getint("reports", "digestDays") * 24 * 60 * 60 for dbreport in getReports(startTime): report = { "guid": dbreport["guid"], "status": dbreport["status"], "url": get_config().get("reports", "urlRoot") + dbreport["guid"] + "#secret=" + calculateReportSecret(dbreport["guid"]), "site": dbreport["site"], "comment": dbreport["comment"], "type": dbreport["type"], "subscriptions": [], "contact": dbreport["contact"], "score": getUserUsefulnessScore(dbreport["contact"]), "hasscreenshot": dbreport["hasscreenshot"], "knownIssues": dbreport["knownissues"], "time": dbreport["ctime"], } recipients = set() reportSubscriptions = getReportSubscriptions(dbreport["guid"]) if dbreport["type"] == "false positive" or dbreport["type"] == "false negative": for subscription in reportSubscriptions: subscriptionID = subscription.get("url", "unknown") # Send false negatives to all subscription authors, false positives # only to subscriptions with matching filters if subscriptionID in subscriptions and ( dbreport["type"] == "false negative" or subscription.get("hasmatches", 0) > 0 ): name, email = parseaddr(subscriptions[subscriptionID].email) if email and not email in recipients: recipients.add(email) emails[email].append(report) report["subscriptions"].append(getSubscriptionInfo(subscriptions[subscriptionID])) else: for subscription in reportSubscriptions: subscriptionID = subscription.get("url", "unknown") report["subscriptions"].append(getSubscriptionInfo(subscriptions[subscriptionID])) recipients.add(defemail) emails[defemail].append(report) # Generate new digests digests = set() for email, reports in emails.iteritems(): if len(reports) == 0: continue file = getDigestPath(dir, email) template = get_template(get_config().get("reports", "htmlDigestTemplate")) template.stream({"email": email, "reports": reports}).dump(file, encoding="utf-8") digests.add(file) # Remove not updated digests which are more then 2 weeks old for filename in os.listdir(dir): file = os.path.join(dir, filename) if ( os.path.isfile(file) and file not in digests and re.match(r"^[\da-f]{32}\.html$", filename) and os.stat(file).st_mtime < currentTime - 14 * 24 * 60 * 60 ): os.remove(file)