def annotation(self, req):
email, path = self.split_path(req, 'annotation')
filename = self.make_filename('annotation', email, path)
if not os.path.exists(filename):
data = {'annotations': []}
else:
with open(filename, 'rb') as fp:
data = json.loads(fp.read())
if req.method == 'GET':
return Response(json=data)
elif req.method == 'POST':
req_data = req.json
if req_data.get('annotations'):
data['annotations'].extend(req_data['annotations'])
if req_data.get('deletes'):
for delete in req_data['deletes']:
for ann in list(data['annotations']):
if ann['id'] == delete['id']:
data['annotations'].remove(ann)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
with open(filename, 'wb') as fp:
fp.write(json.dumps(data))
return Response(json=data)
else:
return exc.HTTPMethodNotAllowed(allow='GET,POST')
def setup(self, req):
resp = Response(content_type='text/plain')
root = req.application_url
section = self.mapper.config['setup']
for key in sorted(section):
url = section.interpolate(key, root=root)
subreq = Request.blank(url)
subresp = subreq.send(self)
resp.write('URL: %s\n' % url)
resp_body = subresp.body or subresp.status
resp_body = resp_body.strip() + '\n'
resp.write(indent(resp_body))
return resp
def register_all(self, req):
resp = Response(content_type='text/plain', body='Registering scripts\n')
dir = os.path.normcase(os.path.abspath(self.dir))
for dirpath, dirnames, filenames in os.walk(dir):
if '.git' in dirnames:
dirnames.remove('.git')
for fn in filenames:
fn = os.path.join(dirpath, fn)
assert fn.startswith(dir)
fn = fn[len(dir):]
assert fn.startswith('/')
url = req.application_url + '/api/scripts' + fn
resp.write(' registering URL: %s\n' % url)
register_url = urlparse.urljoin(req.application_url, self.register_url)
send_request(req, register_url, url)
return resp
def query_consumer(self, req):
type = req.GET['consumer']
matches = []
for url, consumer in self.consumers.iteritems():
if type in consumer['types']:
matches.append(consumer)
return Response(json={'matches': matches})
def describe(self, req):
data = dict(
name='Annotate a document',
sendToPage=req.application_url + '/create.html',
sendToPageFunction='savePage',
types=['html'],
)
return Response(json=data)
def register_all(self, req):
resp = Response(content_type='text/plain',
body='Registering scripts\n')
dir = os.path.normcase(os.path.abspath(self.dir))
for dirpath, dirnames, filenames in os.walk(dir):
if '.git' in dirnames:
dirnames.remove('.git')
for fn in filenames:
fn = os.path.join(dirpath, fn)
assert fn.startswith(dir)
fn = fn[len(dir):]
assert fn.startswith('/')
url = req.application_url + '/api/scripts' + fn
resp.write(' registering URL: %s\n' % url)
register_url = urlparse.urljoin(req.application_url,
self.register_url)
send_request(req, register_url, url)
return resp
def save(self, req):
if not req.email:
return Response(status=403,
content_type='text/plain',
body='Not logged in')
email, path = self.split_path(req, 'save')
if email != req.email:
return Response(status=403,
content_type='text/plain',
body='Email not correct (%r, not %r)' %
(req.email, email))
if req.method != 'PUT':
return exc.HTTPMethodNotAllowed(allow='PUT')
data = req.json
filename = self.make_filename('page', email, path)
write_file(filename, json.dumps(data))
location = req.application_url + '/page/' + urllib.quote(
email) + '/' + urllib.quote(path, '')
return Response(json={'location': location})
def __call__(self, req):
for i, (prefix, match) in enumerate(self.matchers):
if req.path_info == prefix and not req.path_info.endswith('/'):
return Response(status=301, location=req.url + '/')
if req.path_info.startswith(prefix + '/'):
req.script_name += prefix
req.path_info = req.path_info[len(prefix):]
return match
if prefix == '/':
return match
return exc.HTTPNotFound('No route')
def query(self, req):
if req.GET.get('consumer'):
return self.query_consumer(req)
url = req.GET.get('url')
url_domain = urlparse.urlsplit(url).netloc.lower().split(':')[0]
matches = []
transformers = self.transformers
for key, all_data in transformers.iteritems():
for func_data in all_data:
done = False
domains = func_data.get('domain', [])
if isinstance(domains, basestring):
domains = [domains]
for domain in domains:
if (url_domain == domain or domain == '*'
or domain.startswith('.')
and url_domain.endswith(domain)):
matches.append(func_data)
done = True
break
if done:
continue
urls = func_data.get('urls', [])
if isinstance(urls, basestring):
urls = [urls]
for url_match in urls:
if url.startswith(url_match):
matches.append(func_data)
done = True
break
if done:
continue
url_regexes = func_data.get('url-regex', [])
if isinstance(url_regexes, basestring):
url_regexes = [url_regexes]
for url_regex in url_regexes:
if re.match(url_regex, url):
matches.append(func_data)
break
result = {'extractors': matches}
types = set(item['type'] for item in matches if item.get('type'))
consumers = []
for type in types:
for consumer in self.consumers.itervalues():
if type in consumer['types']:
consumers.append(consumer)
result['consumers'] = consumers
return Response(json=result)
def setup(self, req):
resp = Response(content_type='text/plain')
root = req.application_url
section = self.mapper.config['setup']
for key in sorted(section):
url = section.interpolate(key, root=root)
subreq = Request.blank(url)
subresp = subreq.send(self)
resp.write('URL: %s\n' % url)
resp_body = subresp.body or subresp.status
resp_body = resp_body.strip() + '\n'
resp.write(indent(resp_body))
return resp
def register_consumer(self, req):
url = get_url(req)
if not url:
return exc.HTTPBadRequest('No url parameter provided')
body = send_request(req, url)
try:
data = json.loads(body)
except ValueError:
import sys
print >> sys.stderr, 'Bad data for url %s: %r' % (url, body)
raise
if 'post' in data:
data['post'] = urlparse.urljoin(url, data['post'])
if 'sendToPage' in data:
data['sendToPage'] = urlparse.urljoin(url, data['sendToPage'])
data['url'] = url
consumers = self.consumers
consumers[url] = data
self.consumers = consumers
return Response(content_type='text/plain',
body='Added %s at %s' % (data.get('name'), url))
def script_store(self, req):
prefix = '/api/scripts'
assert req.path_info.startswith(prefix)
path_parts = req.path_info[len(prefix):].strip('/').split('/', 1)
email = path_parts[0]
name = path_parts[1]
filename = self.get_script_filename(email, name)
if req.method == 'PUT':
if req.email != email:
return exc.HTTPForbidden()
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
with open(filename, 'wb') as fp:
fp.write(req.body)
return Response('', content_type=None)
elif req.method == 'GET':
if not os.path.exists(filename):
return exc.HTTPNotFound()
return FileApp(filename)
else:
return exc.HTTPMethodNotAllow(allow='PUT,GET')
def page(self, req):
email, path = self.split_path(req, 'page')
filename = self.make_filename('page', email, path)
if not os.path.exists(filename):
return exc.HTTPNotFound()
with open(filename, 'rb') as fp:
data = json.loads(fp.read())
if data['data'].get('bodyAttrs'):
body_attrs = [
' %s="%s"' % (name, cgi.escape(value))
for name, value in data['data']['bodyAttrs'].items()
]
else:
body_attrs = ''
page = self.page_template.substitute(
location=data['location'],
head=data['data']['head'],
application_url=req.application_url,
body_attrs=body_attrs,
body=data['data']['body'],
auth_html=req.get_sub('auth'),
annotation_url=req.url.replace('/page/', '/annotation/'),
)
return Response(page)
class Application(object):
def __init__(self, dir):
self.dir = dir
if not os.path.exists(dir):
os.makedirs(dir)
self.transformer_fn = os.path.join(dir, 'transformers.json')
self.consumers_fn = os.path.join(dir, 'consumers.json')
self.directory_app = ServeStatic(__name__, 'static-repo')
@wsgify
def __call__(self, req):
if req.path_info == '/query':
return self.query(req)
if req.path_info == '/':
req.path_info = '/static/'
if req.path_info == '/register':
return self.register(req)
if req.path_info == '/register-consumer':
return self.register_consumer(req)
if req.path_info == '/check-all':
return self.check_all(req)
if req.path_info_peek() == 'static':
req.path_info_pop()
return self.directory_app
return self.directory_app
return exc.HTTPNotFound()
############################################################
## Query/repository
transformers = JsonFile('transformer_fn')
consumers = JsonFile('consumers_fn')
@wsgify
def query(self, req):
if req.GET.get('consumer'):
return self.query_consumer(req)
url = req.GET.get('url')
url_domain = urlparse.urlsplit(url).netloc.lower().split(':')[0]
matches = []
transformers = self.transformers
for key, all_data in transformers.iteritems():
for func_data in all_data:
done = False
domains = func_data.get('domain', [])
if isinstance(domains, basestring):
domains = [domains]
for domain in domains:
if (url_domain == domain or domain == '*'
or domain.startswith('.')
and url_domain.endswith(domain)):
matches.append(func_data)
done = True
break
if done:
continue
urls = func_data.get('urls', [])
if isinstance(urls, basestring):
urls = [urls]
for url_match in urls:
if url.startswith(url_match):
matches.append(func_data)
done = True
break
if done:
continue
url_regexes = func_data.get('url-regex', [])
if isinstance(url_regexes, basestring):
url_regexes = [url_regexes]
for url_regex in url_regexes:
if re.match(url_regex, url):
matches.append(func_data)
break
result = {'extractors': matches}
types = set(item['type'] for item in matches if item.get('type'))
consumers = []
for type in types:
for consumer in self.consumers.itervalues():
if type in consumer['types']:
consumers.append(consumer)
result['consumers'] = consumers
return Response(json=result)
@wsgify
def query_consumer(self, req):
type = req.GET['consumer']
matches = []
for url, consumer in self.consumers.iteritems():
if type in consumer['types']:
matches.append(consumer)
return Response(json={'matches': matches})
@wsgify
def register(self, req):
js_url = get_url(req)
if not js_url:
return exc.HTTPBadRequest('No url parameter provided')
data = send_request(req, js_url)
try:
properties = parse_metadata(data, js_url)
except ValueError, e:
return exc.HTTPBadRequest(str(e))
transformers = self.transformers
transformers[js_url] = properties
self.transformers = transformers
return Response(content_type='text/plain',
body='Added %s at %s' %
(properties[0].get('name'), js_url))
def check_all(self, req):
resp = Response(content_type='text/plain')
transformers = self.transformers
resp.write('Extractors:\n')
for url in list(transformers):
if '0.0.0.0' in url:
del transformers[url]
continue
try:
send_request(req, url)
except send_request.Error:
del transformers[url]
resp.write(' bad %s\n' % url)
else:
resp.write(' good %s\n' % url)
consumers = self.consumers
resp.write('Consumers:\n')
for url in list(consumers):
if '0.0.0.0' in url:
del consumers[url]
continue
try:
send_request(req, url)
except send_request.Error:
del consumers[url]
resp.write(' bad %s\n' % url)
else:
resp.write(' good %s\n' % url)
if 'commit' in req.GET:
self.transformers = transformers
self.consumers = consumers
resp.write('Bad items removed\n')
return resp
class DispatcherApp(object):
def __init__(self,
secret_filename='/tmp/seeit-services/secret.txt',
config_file='mapper.ini',
**vars):
self._secret_filename = secret_filename
self.static_app = ServeStatic(__name__, 'static-auth', '/static-auth')
self.config_file = os.path.join(
os.path.dirname(os.path.abspath(__file__)), config_file)
self.mapper = Mapper(vars=vars)
self.mapper.add_configs(self.config_file)
@wsgify
def __call__(self, req):
## Another hack for Petri (https://bugzilla.mozilla.org/show_bug.cgi?id=807796)
file_wrapper = None
if 'wsgi.file_wrapper' in req.environ:
file_wrapper = req.environ.pop('wsgi.file_wrapper')
if not file_wrapper:
return self.respond
else:
resp = req.send(self.respond)
req.environ['wsgi.file_wrapper'] = file_wrapper
return resp
@wsgify
def respond(self, req):
## Hack for Petri
if req.headers.get('X-SSL', '').lower() == 'on':
req.scheme = 'https'
self.set_auth(req)
req.root = (req.application_url, self)
if req.path_info == '/auth':
return self.auth(req)
if req.path_info == '/setup':
return self.setup(req)
if self.static_app.matches(req):
return self.static_app
return self.mapper
############################################################
## Auth stuff
def set_auth(self, req):
req.add_sub('auth',
'</body>',
('<script src="https://browserid.org/include.js"></script>'
'<script src="%s/static-auth/auth.js"></script>'
'<script>Auth.authUrl=%r</script>') %
(req.application_url, req.application_url + '/auth'),
replace=False)
auth = req.GET.get('auth')
if not auth:
return
if '.' in auth:
sig, auth = auth.split('.', 1)
if self.signature(auth) == sig:
req.auth = json.loads(auth)
@property
def secret(self):
secret = read_file(self._secret_filename)
if not secret:
secret = make_random(10)
write_file(self._secret_filename, secret)
return secret
def signature(self, text):
return sign(self.secret, text)
@wsgify
def auth(self, req):
try:
assertion = req.params['assertion']
audience = req.params['audience']
except KeyError, e:
return exc.HTTPBadRequest('Missing key: %s' % e)
r = urllib.urlopen(
"https://browserid.org/verify",
urllib.urlencode(dict(assertion=assertion, audience=audience)))
r = json.loads(r.read())
if r['status'] == 'okay':
r['audience'] = audience
static = json.dumps(r)
static = self.signature(static) + '.' + static
r['auth'] = {'query': {'auth': static}}
return Response(json=r)
def check_all(self, req):
resp = Response(content_type='text/plain')
transformers = self.transformers
resp.write('Extractors:\n')
for url in list(transformers):
if '0.0.0.0' in url:
del transformers[url]
continue
try:
send_request(req, url)
except send_request.Error:
del transformers[url]
resp.write(' bad %s\n' % url)
else:
resp.write(' good %s\n' % url)
consumers = self.consumers
resp.write('Consumers:\n')
for url in list(consumers):
if '0.0.0.0' in url:
del consumers[url]
continue
try:
send_request(req, url)
except send_request.Error:
del consumers[url]
resp.write(' bad %s\n' % url)
else:
resp.write(' good %s\n' % url)
if 'commit' in req.GET:
self.transformers = transformers
self.consumers = consumers
resp.write('Bad items removed\n')
return resp
def fill_prefill(self, req):
resp = Response(content_type='text/plain',
body='Filling prefill from %s\n' % self.dir)
force = req.params.get('force', True)
return self.copy_files(resp, force, self.dir, self.prefill_dir)
