def annotation(self, req): email, path = self.split_path(req, 'annotation') filename = self.make_filename('annotation', email, path) if not os.path.exists(filename): data = {'annotations': []} else: with open(filename, 'rb') as fp: data = json.loads(fp.read()) if req.method == 'GET': return Response(json=data) elif req.method == 'POST': req_data = req.json if req_data.get('annotations'): data['annotations'].extend(req_data['annotations']) if req_data.get('deletes'): for delete in req_data['deletes']: for ann in list(data['annotations']): if ann['id'] == delete['id']: data['annotations'].remove(ann) if not os.path.exists(os.path.dirname(filename)): os.makedirs(os.path.dirname(filename)) with open(filename, 'wb') as fp: fp.write(json.dumps(data)) return Response(json=data) else: return exc.HTTPMethodNotAllowed(allow='GET,POST')
def check_all(self, req): resp = Response(content_type='text/plain') transformers = self.transformers resp.write('Extractors:\n') for url in list(transformers): if '0.0.0.0' in url: del transformers[url] continue try: send_request(req, url) except send_request.Error: del transformers[url] resp.write(' bad %s\n' % url) else: resp.write(' good %s\n' % url) consumers = self.consumers resp.write('Consumers:\n') for url in list(consumers): if '0.0.0.0' in url: del consumers[url] continue try: send_request(req, url) except send_request.Error: del consumers[url] resp.write(' bad %s\n' % url) else: resp.write(' good %s\n' % url) if 'commit' in req.GET: self.transformers = transformers self.consumers = consumers resp.write('Bad items removed\n') return resp
def query_consumer(self, req): type = req.GET['consumer'] matches = [] for url, consumer in self.consumers.iteritems(): if type in consumer['types']: matches.append(consumer) return Response(json={'matches': matches})
def describe(self, req): data = dict( name='Annotate a document', sendToPage=req.application_url + '/create.html', sendToPageFunction='savePage', types=['html'], ) return Response(json=data)
def save(self, req): if not req.email: return Response(status=403, content_type='text/plain', body='Not logged in') email, path = self.split_path(req, 'save') if email != req.email: return Response(status=403, content_type='text/plain', body='Email not correct (%r, not %r)' % (req.email, email)) if req.method != 'PUT': return exc.HTTPMethodNotAllowed(allow='PUT') data = req.json filename = self.make_filename('page', email, path) write_file(filename, json.dumps(data)) location = req.application_url + '/page/' + urllib.quote( email) + '/' + urllib.quote(path, '') return Response(json={'location': location})
def __call__(self, req): for i, (prefix, match) in enumerate(self.matchers): if req.path_info == prefix and not req.path_info.endswith('/'): return Response(status=301, location=req.url + '/') if req.path_info.startswith(prefix + '/'): req.script_name += prefix req.path_info = req.path_info[len(prefix):] return match if prefix == '/': return match return exc.HTTPNotFound('No route')
def setup(self, req): resp = Response(content_type='text/plain') root = req.application_url section = self.mapper.config['setup'] for key in sorted(section): url = section.interpolate(key, root=root) subreq = Request.blank(url) subresp = subreq.send(self) resp.write('URL: %s\n' % url) resp_body = subresp.body or subresp.status resp_body = resp_body.strip() + '\n' resp.write(indent(resp_body)) return resp
def query(self, req): if req.GET.get('consumer'): return self.query_consumer(req) url = req.GET.get('url') url_domain = urlparse.urlsplit(url).netloc.lower().split(':')[0] matches = [] transformers = self.transformers for key, all_data in transformers.iteritems(): for func_data in all_data: done = False domains = func_data.get('domain', []) if isinstance(domains, basestring): domains = [domains] for domain in domains: if (url_domain == domain or domain == '*' or domain.startswith('.') and url_domain.endswith(domain)): matches.append(func_data) done = True break if done: continue urls = func_data.get('urls', []) if isinstance(urls, basestring): urls = [urls] for url_match in urls: if url.startswith(url_match): matches.append(func_data) done = True break if done: continue url_regexes = func_data.get('url-regex', []) if isinstance(url_regexes, basestring): url_regexes = [url_regexes] for url_regex in url_regexes: if re.match(url_regex, url): matches.append(func_data) break result = {'extractors': matches} types = set(item['type'] for item in matches if item.get('type')) consumers = [] for type in types: for consumer in self.consumers.itervalues(): if type in consumer['types']: consumers.append(consumer) result['consumers'] = consumers return Response(json=result)
def register_all(self, req): resp = Response(content_type='text/plain', body='Registering scripts\n') dir = os.path.normcase(os.path.abspath(self.dir)) for dirpath, dirnames, filenames in os.walk(dir): if '.git' in dirnames: dirnames.remove('.git') for fn in filenames: fn = os.path.join(dirpath, fn) assert fn.startswith(dir) fn = fn[len(dir):] assert fn.startswith('/') url = req.application_url + '/api/scripts' + fn resp.write(' registering URL: %s\n' % url) register_url = urlparse.urljoin(req.application_url, self.register_url) send_request(req, register_url, url) return resp
def register_consumer(self, req): url = get_url(req) if not url: return exc.HTTPBadRequest('No url parameter provided') body = send_request(req, url) try: data = json.loads(body) except ValueError: import sys print >> sys.stderr, 'Bad data for url %s: %r' % (url, body) raise if 'post' in data: data['post'] = urlparse.urljoin(url, data['post']) if 'sendToPage' in data: data['sendToPage'] = urlparse.urljoin(url, data['sendToPage']) data['url'] = url consumers = self.consumers consumers[url] = data self.consumers = consumers return Response(content_type='text/plain', body='Added %s at %s' % (data.get('name'), url))
def script_store(self, req): prefix = '/api/scripts' assert req.path_info.startswith(prefix) path_parts = req.path_info[len(prefix):].strip('/').split('/', 1) email = path_parts[0] name = path_parts[1] filename = self.get_script_filename(email, name) if req.method == 'PUT': if req.email != email: return exc.HTTPForbidden() if not os.path.exists(os.path.dirname(filename)): os.makedirs(os.path.dirname(filename)) with open(filename, 'wb') as fp: fp.write(req.body) return Response('', content_type=None) elif req.method == 'GET': if not os.path.exists(filename): return exc.HTTPNotFound() return FileApp(filename) else: return exc.HTTPMethodNotAllow(allow='PUT,GET')
def page(self, req): email, path = self.split_path(req, 'page') filename = self.make_filename('page', email, path) if not os.path.exists(filename): return exc.HTTPNotFound() with open(filename, 'rb') as fp: data = json.loads(fp.read()) if data['data'].get('bodyAttrs'): body_attrs = [ ' %s="%s"' % (name, cgi.escape(value)) for name, value in data['data']['bodyAttrs'].items() ] else: body_attrs = '' page = self.page_template.substitute( location=data['location'], head=data['data']['head'], application_url=req.application_url, body_attrs=body_attrs, body=data['data']['body'], auth_html=req.get_sub('auth'), annotation_url=req.url.replace('/page/', '/annotation/'), ) return Response(page)
class Application(object): def __init__(self, dir): self.dir = dir if not os.path.exists(dir): os.makedirs(dir) self.transformer_fn = os.path.join(dir, 'transformers.json') self.consumers_fn = os.path.join(dir, 'consumers.json') self.directory_app = ServeStatic(__name__, 'static-repo') @wsgify def __call__(self, req): if req.path_info == '/query': return self.query(req) if req.path_info == '/': req.path_info = '/static/' if req.path_info == '/register': return self.register(req) if req.path_info == '/register-consumer': return self.register_consumer(req) if req.path_info == '/check-all': return self.check_all(req) if req.path_info_peek() == 'static': req.path_info_pop() return self.directory_app return self.directory_app return exc.HTTPNotFound() ############################################################ ## Query/repository transformers = JsonFile('transformer_fn') consumers = JsonFile('consumers_fn') @wsgify def query(self, req): if req.GET.get('consumer'): return self.query_consumer(req) url = req.GET.get('url') url_domain = urlparse.urlsplit(url).netloc.lower().split(':')[0] matches = [] transformers = self.transformers for key, all_data in transformers.iteritems(): for func_data in all_data: done = False domains = func_data.get('domain', []) if isinstance(domains, basestring): domains = [domains] for domain in domains: if (url_domain == domain or domain == '*' or domain.startswith('.') and url_domain.endswith(domain)): matches.append(func_data) done = True break if done: continue urls = func_data.get('urls', []) if isinstance(urls, basestring): urls = [urls] for url_match in urls: if url.startswith(url_match): matches.append(func_data) done = True break if done: continue url_regexes = func_data.get('url-regex', []) if isinstance(url_regexes, basestring): url_regexes = [url_regexes] for url_regex in url_regexes: if re.match(url_regex, url): matches.append(func_data) break result = {'extractors': matches} types = set(item['type'] for item in matches if item.get('type')) consumers = [] for type in types: for consumer in self.consumers.itervalues(): if type in consumer['types']: consumers.append(consumer) result['consumers'] = consumers return Response(json=result) @wsgify def query_consumer(self, req): type = req.GET['consumer'] matches = [] for url, consumer in self.consumers.iteritems(): if type in consumer['types']: matches.append(consumer) return Response(json={'matches': matches}) @wsgify def register(self, req): js_url = get_url(req) if not js_url: return exc.HTTPBadRequest('No url parameter provided') data = send_request(req, js_url) try: properties = parse_metadata(data, js_url) except ValueError, e: return exc.HTTPBadRequest(str(e)) transformers = self.transformers transformers[js_url] = properties self.transformers = transformers return Response(content_type='text/plain', body='Added %s at %s' % (properties[0].get('name'), js_url))
def fill_prefill(self, req): resp = Response(content_type='text/plain', body='Filling prefill from %s\n' % self.dir) force = req.params.get('force', True) return self.copy_files(resp, force, self.dir, self.prefill_dir)
class DispatcherApp(object): def __init__(self, secret_filename='/tmp/seeit-services/secret.txt', config_file='mapper.ini', **vars): self._secret_filename = secret_filename self.static_app = ServeStatic(__name__, 'static-auth', '/static-auth') self.config_file = os.path.join( os.path.dirname(os.path.abspath(__file__)), config_file) self.mapper = Mapper(vars=vars) self.mapper.add_configs(self.config_file) @wsgify def __call__(self, req): ## Another hack for Petri (https://bugzilla.mozilla.org/show_bug.cgi?id=807796) file_wrapper = None if 'wsgi.file_wrapper' in req.environ: file_wrapper = req.environ.pop('wsgi.file_wrapper') if not file_wrapper: return self.respond else: resp = req.send(self.respond) req.environ['wsgi.file_wrapper'] = file_wrapper return resp @wsgify def respond(self, req): ## Hack for Petri if req.headers.get('X-SSL', '').lower() == 'on': req.scheme = 'https' self.set_auth(req) req.root = (req.application_url, self) if req.path_info == '/auth': return self.auth(req) if req.path_info == '/setup': return self.setup(req) if self.static_app.matches(req): return self.static_app return self.mapper ############################################################ ## Auth stuff def set_auth(self, req): req.add_sub('auth', '</body>', ('<script src="https://browserid.org/include.js"></script>' '<script src="%s/static-auth/auth.js"></script>' '<script>Auth.authUrl=%r</script>') % (req.application_url, req.application_url + '/auth'), replace=False) auth = req.GET.get('auth') if not auth: return if '.' in auth: sig, auth = auth.split('.', 1) if self.signature(auth) == sig: req.auth = json.loads(auth) @property def secret(self): secret = read_file(self._secret_filename) if not secret: secret = make_random(10) write_file(self._secret_filename, secret) return secret def signature(self, text): return sign(self.secret, text) @wsgify def auth(self, req): try: assertion = req.params['assertion'] audience = req.params['audience'] except KeyError, e: return exc.HTTPBadRequest('Missing key: %s' % e) r = urllib.urlopen( "https://browserid.org/verify", urllib.urlencode(dict(assertion=assertion, audience=audience))) r = json.loads(r.read()) if r['status'] == 'okay': r['audience'] = audience static = json.dumps(r) static = self.signature(static) + '.' + static r['auth'] = {'query': {'auth': static}} return Response(json=r)