def test_private_project_read__reader_only(self):
with self.app.test_request_context():
project, permission = _project_and_permission(private=True,
reader=True)
flask.session['id'] = 1
self.app.preprocess_request()
self.assertTrue(authz.project_read(project))
def suggest_property(slug):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
prefix = '%%%s%%' % request.args.get('prefix', '')
log.info("Suggesting property names in %s: %r", project.slug, prefix)
q = db.session.query(Attribute)
q = q.join(Schema)
q = q.filter(Schema.obj == 'entity')
q = q.filter(Schema.project == project)
q = q.filter(or_(Attribute.label.ilike(prefix),
Attribute.name.ilike(prefix)))
q = q.limit(get_limit(default=5))
matches = []
for attribute in q:
matches.append({
'name': attribute.label,
'n:type': {
'id': '/properties/property',
'name': 'Property'
},
'id': attribute.name
})
return jsonify({
"code": "/api/status/ok",
"status": "200 OK",
"prefix": request.args.get('prefix', ''),
"result": matches
})
def view(slug, name):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
if not project.private:
validate_cache(last_modified=project.updated_at)
schema = object_or_404(Schema.by_name(project, name))
return jsonify(schema)
def suggest_property(slug):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
prefix = "%%%s%%" % request.args.get("prefix", "")
log.info("Suggesting property names in %s: %r", project.slug, prefix)
q = db.session.query(Attribute)
q = q.join(Schema)
q = q.filter(Schema.obj == "entity")
q = q.filter(Schema.project == project)
q = q.filter(or_(Attribute.label.ilike(prefix), Attribute.name.ilike(prefix)))
q = q.limit(get_limit(default=5))
matches = []
for attribute in q:
matches.append(
{
"name": attribute.label,
"n:type": {"id": "/properties/property", "name": "Property"},
"id": attribute.name,
}
)
return jsonify(
{"code": "/api/status/ok", "status": "200 OK", "prefix": request.args.get("prefix", ""), "result": matches}
)
def suggest_type(slug):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
prefix = '%%%s%%' % request.args.get('prefix', '')
log.info("Suggesting types in %s: %r", project.slug, prefix)
q = db.session.query(Schema)
q = q.filter(Schema.obj == 'entity')
q = q.filter(Schema.hidden == False) # noqa
q = q.filter(Schema.project == project)
q = q.filter(or_(Schema.label.ilike(prefix), Schema.name.ilike(prefix)))
q = q.limit(get_limit(default=5))
matches = []
for schema in q:
matches.append({
'name': schema.label,
'id': '/%s/%s' % (slug, schema.name)
})
return jsonify({
"code": "/api/status/ok",
"status": "200 OK",
"prefix": request.args.get('prefix', ''),
"result": matches
})
def view(slug, name):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
if not project.private:
validate_cache(last_modified=project.updated_at)
schema = object_or_404(Schema.by_name(project, name))
return jsonify(schema)
def reconcile(slug):
"""
Reconciliation API, emulates Google Refine API. See:
http://code.google.com/p/google-refine/wiki/ReconciliationServiceApi
"""
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
# TODO: Add proper support for types and namespacing.
data = request.args.copy()
data.update(request.form.copy())
if 'query' in data:
# single
q = data.get('query')
if q.startswith('{'):
try:
q = json.loads(q)
except ValueError:
raise BadRequest()
else:
q = data
return jsonify(reconcile_op(project, q))
elif 'queries' in data:
# multiple requests in one query
qs = data.get('queries')
try:
qs = json.loads(qs)
except ValueError:
raise BadRequest()
queries = {}
for k, q in qs.items():
queries[k] = reconcile_op(project, q)
return jsonify(queries)
else:
return reconcile_index(project)
def test_private_project_read__reader_only(self):
with self.app.test_request_context():
project, permission = _project_and_permission(
private=True, reader=True)
flask.session['id'] = 1
self.app.preprocess_request()
self.assertTrue(authz.project_read(project))
def suggest_property(slug):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
prefix = '%%%s%%' % request.args.get('prefix', '')
q = db.session.query(Attribute)
q = q.join(Schema)
q = q.filter(Schema.obj=='entity')
q = q.filter(Schema.project==project)
q = q.filter(or_(Attribute.label.ilike(prefix), Attribute.name.ilike(prefix)))
q = q.limit(get_limit(default=5))
matches = []
for attribute in q:
matches.append({
'name': attribute.label,
'n:type': {
'id': '/properties/property',
'name': 'Property'
},
'id': attribute.name
})
return jsonify({
"code" : "/api/status/ok",
"status" : "200 OK",
"prefix" : request.args.get('prefix', ''),
"result" : matches
})
def index(slug):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
validate_cache(last_modified=project.updated_at)
query = Schema.all()
query = query.filter_by(project=project)
pager = Pager(query, slug=slug)
return jsonify(pager, index=not arg_bool("full"))
def index(slug):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
validate_cache(last_modified=project.updated_at)
query = Schema.all()
query = query.filter_by(project=project)
pager = Pager(query, slug=slug)
return jsonify(pager, index=not arg_bool('full'))
def test_private_project_read__admin_only(self):
"""Perms are designed such that admin/editor cannot read private
projects unless they have explicit read permission."""
project, permission = _project_and_permission(private=True, admin=True)
with self.app.test_request_context():
flask.session['id'] = 1
self.app.preprocess_request()
self.assertFalse(authz.project_read(project))
def aliases(slug):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
sio = StringIO()
export_aliases(project, sio)
sio.seek(0)
res = send_file(sio, mimetype='text/csv')
res.headers['Content-Disposition'] = 'filename=%s-aliases.csv' % project.slug
return res
def serve(id):
file = object_or_404(File.by_id(id))
authz.require(authz.project_read(file.project))
sio = StringIO()
sio.write(file.data)
sio.seek(0)
res = send_file(sio, mimetype=file.mime_type)
res.headers['Content-Disposition'] = 'filename=%s' % file.file_name
return res
def test_private_project_read__admin_only(self):
"""Perms are designed such that admin/editor cannot read private
projects unless they have explicit read permission."""
project, permission = _project_and_permission(
private=True, admin=True)
with self.app.test_request_context():
flask.session['id'] = 1
self.app.preprocess_request()
self.assertFalse(authz.project_read(project))
def serve(id):
file = object_or_404(File.by_id(id))
authz.require(authz.project_read(file.project))
sio = StringIO()
sio.write(file.data)
sio.seek(0)
res = send_file(sio, mimetype=file.mime_type)
res.headers['Content-Disposition'] = 'filename=%s' % file.file_name
return res
def graph(slug):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
extractor = GraphExtractor(project_id=project.id)
if not project.private:
validate_cache(keys=extractor.to_hash())
if extractor.format == 'gexf':
return Response(extractor.to_gexf(), mimetype='text/xml')
return jsonify(extractor)
def graph(id):
entity = object_or_404(Entity.by_id(id))
authz.require(authz.project_read(entity.project))
extractor = GraphExtractor(root_id=entity.id)
validate_cache(keys=extractor.to_hash())
if extractor.format == 'gexf':
return Response(extractor.to_gexf(),
mimetype='text/xml')
return jsonify(extractor.to_dict())
def index(slug):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
validate_cache(last_modified=project.updated_at)
query = Schema.all()
query = query.filter_by(project=project)
pager = Pager(query)
conv = lambda es: [schemata.to_rest_index(e) for e in es]
return jsonify(pager.to_dict(conv))
def graph(slug):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
extractor = GraphExtractor(project_id=project.id)
if not project.private:
validate_cache(keys=extractor.to_hash())
if extractor.format == 'gexf':
return Response(extractor.to_gexf(),
mimetype='text/xml')
return jsonify(extractor.to_dict())
def suggest_entity(slug):
"""
Suggest API, emulates Google Refine API. See:
https://github.com/OpenRefine/OpenRefine/wiki/Reconciliation-Service-API
"""
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
prefix = "%%%s%%" % request.args.get("prefix", "")
log.info("Suggesting entities in %s: %r", project.slug, prefix)
q = db.session.query(Entity)
q = q.join(Property)
q = q.join(Project)
q = q.filter(Property.name == "name")
q = q.filter(Property.active == True)
q = q.filter(Property.entity_id == Entity.id)
q = q.filter(Property.value_string.ilike(prefix))
q = q.filter(Project.slug == slug)
if "type" in request.args:
schema_name = request.args.get("type")
if "/" in schema_name:
_, schema_name = schema_name.rsplit("/", 1)
q = q.join(Schema)
q = q.filter(Schema.name == schema_name)
q = q.distinct()
q = q.limit(get_limit(default=5))
matches = []
for e in q:
data = {
"name": e["name"].value,
"n:type": {},
"type": [],
"uri": url_for("entities_api.view", id=e.id, _external=True),
"id": e.id,
}
for schema in e.schemata:
if schema.hidden:
continue
data["type"].append({"id": "/" + project.slug + "/" + schema.name, "name": schema.label})
if len(data["type"]):
data["n:type"] = data["type"][0]
matches.append(data)
return jsonify(
{"code": "/api/status/ok", "status": "200 OK", "prefix": request.args.get("prefix", ""), "result": matches}
)
def suggest_entity(slug):
"""
Suggest API, emulates Google Refine API. See:
https://github.com/OpenRefine/OpenRefine/wiki/Reconciliation-Service-API
"""
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
prefix = '%%%s%%' % request.args.get('prefix', '')
log.info("Suggesting entities in %s: %r", project.slug, prefix)
q = db.session.query(Entity)
q = q.join(Property)
q = q.join(Project)
q = q.filter(Property.name == 'name')
q = q.filter(Property.active == True) # noqa
q = q.filter(Property.entity_id == Entity.id)
q = q.filter(Property.value_string.ilike(prefix))
q = q.filter(Project.slug == slug)
if 'type' in request.args:
schema_name = request.args.get('type')
if '/' in schema_name:
_, schema_name = schema_name.rsplit('/', 1)
q = q.join(Schema)
q = q.filter(Schema.name == schema_name)
q = q.distinct()
q = q.limit(get_limit(default=5))
matches = []
for e in q:
data = {
'name': e['name'].value,
'n:type': {
'id': '/' + project.slug + '/' + e.schema.name,
'name': e.schema.label
},
'uri': url_for('entities_api.view', id=e.id, _external=True),
'id': e.id
}
data['type'] = [data.get('n:type')]
matches.append(data)
return jsonify({
"code": "/api/status/ok",
"status": "200 OK",
"prefix": request.args.get('prefix', ''),
"result": matches
})
def index(pipeline_id):
pipeline = object_or_404(Pipeline.by_id(pipeline_id))
authz.require(authz.project_read(pipeline.project))
query = LogEntry.all()
query = query.filter(LogEntry.pipeline==pipeline)
if request.args.get('level'):
query = query.filter(LogEntry.level==request.args.get('level'))
pager = Pager(query)
validate_cache(keys=pager.cache_keys())
return jsonify(pager, index=True)
def _index(query, obj):
authz.require(authz.project_read(obj.project))
active_only = arg_bool('active', default=True)
if active_only:
query = query.filter_by(active=True)
if 'name' in request.args:
query = query.filter_by(name=request.args.get('name'))
query = query.order_by(Property.created_at.desc())
pager = Pager(query, obj_id=obj.id)
validate_cache(keys=pager.cache_keys())
return jsonify(pager, index=False)
def suggest_entity(slug):
"""
Suggest API, emulates Google Refine API. See:
https://github.com/OpenRefine/OpenRefine/wiki/Reconciliation-Service-API
"""
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
prefix = request.args.get('prefix', '') + '%'
q = db.session.query(EntityProperty)
q = q.join(Entity)
q = q.join(Project)
q = q.filter(EntityProperty.name=='name')
q = q.filter(EntityProperty.active==True)
q = q.filter(EntityProperty.entity_id!=None)
q = q.filter(EntityProperty.value_string.ilike(prefix))
q = q.filter(Project.slug==slug)
if 'type' in request.args:
schema_name = request.args.get('type')
if '/' in schema_name:
_, schema_name = schema_name.rsplit('/', 1)
q = q.join(Schema)
q = q.filter(Schema.name==schema_name)
q = q.distinct()
q = q.limit(get_limit(default=5))
matches = []
for eprop in q:
matches.append({
'name': eprop.value_string,
'n:type': {
'id': '/' + project.slug,
'name': project.label
},
'id': eprop.entity_id
})
return jsonify({
"code" : "/api/status/ok",
"status" : "200 OK",
"prefix" : request.args.get('prefix', ''),
"result" : matches
})
def query(slug):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
if request.method == 'POST':
query = request.json
else:
try:
query = json.loads(request.args.get('query', 'null'))
assert query is not None
except (TypeError, ValueError, AssertionError):
raise BadRequest('Invalid data submitted')
eq = run_query(project, query)
res = {
'status': 'ok',
'query': eq.node,
'results': eq.run(),
'total': eq.count()
}
return jsonify(res)
def reconcile(slug):
"""
Reconciliation API, emulates Google Refine API. See:
http://code.google.com/p/google-refine/wiki/ReconciliationServiceApi
"""
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
# TODO: Add proper support for types and namespacing.
data = request.args.copy()
data.update(request.form.copy())
if 'query' in data:
# single
q = data.get('query')
if q.startswith('{'):
try:
q = json.loads(q)
except ValueError:
raise BadRequest()
else:
q = data
return jsonify(reconcile_op(project, q))
elif 'queries' in data:
# multiple requests in one query
qs = data.get('queries')
try:
qs = json.loads(qs)
except ValueError:
raise BadRequest()
queries = {}
for k, q in qs.items():
queries[k] = reconcile_op(project, q)
return jsonify(queries)
else:
return reconcile_index(project)
def test_public_project_read(self):
project, permission = _project_and_permission()
with self.app.test_request_context():
flask.session['id'] = 1
self.app.preprocess_request()
self.assertTrue(authz.project_read(project))
def view(id):
entity = object_or_404(Entity.by_id(id))
authz.require(authz.project_read(entity.project))
return jsonify(entities.to_rest(entity))
def view(id):
pipeline = object_or_404(Pipeline.by_id(id))
authz.require(authz.project_read(pipeline.project))
return jsonify(pipeline)
def view(id):
relation = object_or_404(Relation.by_id(id))
authz.require(authz.project_read(relation.project))
return jsonify(relation)
def test_public_project_read(self):
project, permission = _project_and_permission()
with self.app.test_request_context():
flask.session['id'] = 1
self.app.preprocess_request()
self.assertTrue(authz.project_read(project))
def view(slug):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
if not project.private:
validate_cache(last_modified=project.updated_at)
return jsonify(projects.to_rest(project))
def view(id):
file = object_or_404(File.by_id(id))
authz.require(authz.project_read(file.project))
return jsonify(file)
def table(id):
file = object_or_404(File.by_id(id))
authz.require(authz.project_read(file.project))
limit = get_limit(10)
validate_cache(keys={'id': file.id, 'limit': limit})
return jsonify(files.as_table(file, limit))
def table(id):
file = object_or_404(File.by_id(id))
authz.require(authz.project_read(file.project))
limit = get_limit(10)
validate_cache(keys={'id': file.id, 'limit': limit})
return jsonify(files.as_table(file, limit))
def view(id):
file = object_or_404(File.by_id(id))
authz.require(authz.project_read(file.project))
return jsonify(file)
def view(id):
pipeline = object_or_404(Pipeline.by_id(id))
authz.require(authz.project_read(pipeline.project))
return jsonify(pipeline)
def view(pipeline_id, id):
pipeline = object_or_404(Pipeline.by_id(pipeline_id))
authz.require(authz.project_read(pipeline.project))
log_entry = object_or_404(LogEntry.by_id(id))
return jsonify(log_entry)
def test_private_project_read__no_perm(self):
project, permission = _project_and_permission(private=True)
with self.app.test_request_context():
flask.session['id'] = 1
self.app.preprocess_request()
self.assertFalse(authz.project_read(project))
def test_private_project_read__no_perm(self):
project, permission = _project_and_permission(private=True)
with self.app.test_request_context():
flask.session['id'] = 1
self.app.preprocess_request()
self.assertFalse(authz.project_read(project))
