def object_to_dict(obj):
d = {}
for f in obj._meta.fields:
d[f.column] = getattr(obj, f.column)
if not type(d[f.column]) in (long, int, float, bool, types.NoneType):
d[f.column] = unicode(d[f.column])
if type(f) == ForeignKey:
v = getattr(obj, f.name)
if v is not None:
model_name = model_registry.get_name(v.__class__)
d[f.column] = {
'model': model_name,
'id': v.pk,
'__str__': str(v)
}
try:
d[f.column]['url'] = model_registry.get_manager(model_name).url_of(v)
except:
pass
if 'id' not in d:
pk = obj.pk
if not type(pk) in (long, int, float, bool, types.NoneType):
d['id'] = unicode(pk)
else:
d['id'] = pk
return d
def object_to_dict(obj):
d = {}
for f in obj._meta.fields:
d[f.column] = getattr(obj, f.column)
if not type(
d[f.column]) in (long, int, float, bool, types.NoneType):
d[f.column] = unicode(d[f.column])
if type(f) == ForeignKey:
v = getattr(obj, f.name)
if v is not None:
model_name = model_registry.get_name(v.__class__)
d[f.column] = {
'model': model_name,
'id': v.pk,
'__str__': str(v)
}
try:
d[f.column]['url'] = model_registry.get_manager(
model_name).url_of(v)
except:
pass
if 'id' not in d:
pk = obj.pk
if not type(pk) in (long, int, float, bool, types.NoneType):
d['id'] = unicode(pk)
else:
d['id'] = pk
return d
def setUp(self):
blog = Blog(name='Databases')
blog.save()
self.blogs = [blog]
authors = ('John Smith', 'Jane Doe', 'Joe Plummer')
headlines = ('MySQL is a relational DB',
'Postgres is a really good relational DB',
'Neo4J is a graph DB')
self.entries = [
Entry(headline=headline, blog=blog) for headline in headlines
]
for entry in self.entries:
entry.save()
self.authors = [Author(name=name) for name in authors]
for author in self.authors:
author.save()
for i, entry in enumerate(self.entries):
entry.authors.add(self.authors[i])
entry.authors.add(self.authors[(i + 1) % len(self.authors)])
model_registry.register(curious_tests.models)
model_registry.get_manager('Blog').allowed_relationships = ['authors']
def get(self, request, model_name):
try:
cls = model_registry.get_manager(model_name).model_class
except Exception as e:
return self._error(404,
"Unknown model '%s': %s" % (model_name, str(e)))
return self._return(200, ModelView.model_to_dict(cls))
def setUp(self):
names = ('Databases', 'Relational Databases', 'Graph Databases')
authors = ('John Smith', 'Jane Doe', 'Joe Plummer')
headlines = ('MySQL is a relational DB',
'Postgres is a really good relational DB',
'Neo4J is a graph DB')
self.blogs = [Blog(name=name) for name in names]
for blog in self.blogs:
blog.save()
self.entries = [Entry(headline=headline, blog=blog) for headline, blog in zip(headlines, self.blogs)]
for entry in self.entries:
entry.save()
self.authors = [Author(name=name) for name in authors]
for author in self.authors:
author.save()
for i, entry in enumerate(self.entries):
entry.authors.add(self.authors[i])
entry.authors.add(self.authors[(i+1)%len(self.authors)])
model_registry.register(curious_tests.models)
model_registry.get_manager('Blog').allowed_relationships = ['authors']
def post(self, request, model_name):
"""
Fetch objects in batch.
"""
try:
data = json.loads(request.body)
except:
return self._error(400, "Bad data")
if 'ids' not in data:
return self._error(400, "Missing ids array")
app = None
if 'app' in data:
app = data['app']
try:
cls = model_registry.get_manager(model_name).model_class
except:
return self._error(404, "Unknown model '%s'" % model_name)
ignore_excludes = get_param_value(data, 'x', False)
r = ModelView.get_objects_as_json(cls, data['ids'], ignore_excludes, True, True, app)
return self._return(200, r)
def model_to_dict(cls):
model_name = model_registry.get_name(cls)
d = dict(model=model_name, relationships=[])
for f in dir(cls):
if model_registry.get_manager(model_name).is_rel_allowed(f):
d['relationships'].append(f)
return d
def test_property_fields(self):
person_manager = model_registry.get_manager('Person')
person_manager.property_fields = [
'example_property_field',
]
r = self.client.post(
'/curious/q/',
data=json.dumps({
'q': 'Person({0.pk})'.format(self.person),
'd': 1
}),
content_type='application/json',
)
self.assertEquals(r.status_code, 200)
result = json.loads(r.content)['result']
print result
data = json.loads(r.content)['result']['data'][0]
self.assertItemsEqual(
data['fields'],
['id', 'alive', 'example_property_field', 'gender'])
self.assertItemsEqual(data['objects'][0], [
self.person.pk,
self.person.alive,
self.person.example_property_field,
self.person.gender,
])
def model_to_dict(cls):
model_name = model_registry.get_name(cls)
d = dict(model=model_name, relationships=[])
for f in dir(cls):
if model_registry.get_manager(model_name).is_rel_allowed(f):
d['relationships'].append(f)
return d
def post(self, request, model_name):
"""
Fetch objects in batch.
"""
try:
data = json.loads(request.body)
except:
return self._error(400, "Bad data")
if 'ids' not in data:
return self._error(400, "Missing ids array")
app = None
if 'app' in data:
app = data['app']
try:
cls = model_registry.get_manager(model_name).model_class
except:
return self._error(404, "Unknown model '%s'" % model_name)
ignore_excludes = get_param_value(data, 'x', False)
r = ModelView.get_objects_as_json(cls, data['ids'], ignore_excludes,
True, True, app)
return self._return(200, r)
def __get_objects(self):
"""
Get initial objects from object query.
"""
model = self.__obj_query['model']
method = self.__obj_query['method']
filters = self.__obj_query['filters']
filter_f = mk_filter_function(filters)
if method is None:
cls = model_registry.get_manager(model).model_class
q = cls.objects.all()
q = filter_f(q)
return q
else:
f = model_registry.get_manager(model).getattr(method)
return f(filter_f)
def __get_objects(self):
"""
Get initial objects from object query.
"""
model = self.__obj_query['model']
method = self.__obj_query['method']
filters = self.__obj_query['filters']
filter_f = mk_filter_function(filters)
if method is None:
cls = model_registry.get_manager(model).model_class
q = cls.objects.all()
q = filter_f(q)
return q
else:
f = model_registry.get_manager(model).getattr(method)
return f(filter_f)
def get(self, request, model_name, id):
try:
cls = model_registry.get_manager(model_name).model_class
except:
return self._error(404, "Unknown model '%s'" % model_name)
try:
obj = cls.objects.get(pk=id)
except:
return self._error(404, "Cannot find instance '%s' on '%s'" % (id, model_name))
return self._return(200, ObjectView.object_to_dict(obj))
def _process(self, params):
if 'q' not in params:
return self._error(400, 'Missing query')
q = params['q']
check_only = get_param_value(params, 'c', False)
load_data = get_param_value(params, 'd', False)
follow_fk = get_param_value(params, 'fk', True)
ignore_excludes = get_param_value(params, 'x', False)
force = get_param_value(params, 'r', False)
force_cache = get_param_value(params, 'fc', False)
app = params['app'] if 'app' in params else None
try:
query = Query(q)
except Exception as e:
import traceback
traceback.print_exc()
return self._error(400, str(e))
# check query mode
if check_only:
return self._return(200, dict(query=q))
try:
results = self.get_query_results(query, force, force_cache, app)
except Exception as e:
import traceback
traceback.print_exc()
return self._error(400, str(e))
t = datetime.now() - results['computed_on']
if t.seconds > 300:
results['computed_since'] = str(naturaltime(
results['computed_on']))
results['computed_on'] = str(results['computed_on'])
# data mode
objects = []
if load_data:
for result in results['results']:
if result['model']:
model = model_registry.get_manager(
result['model']).model_class
ids = [t[0] for t in result['objects']]
objs = ModelView.get_objects_as_json(
model, ids, ignore_excludes, follow_fk, force, app)
objects.append(objs)
else:
objects.append([])
results['data'] = objects
# print results
return self._return(200, results)
def _validate(query):
"""
Validate a query. A query is an array whose elements are model
relationships or subqueries. This function checks each model relationship
to make sure the model and the relationship exist.
"""
for rel in query:
if 'orquery' in rel:
for q in rel['orquery']:
Query._validate(q)
elif 'subquery' in rel:
Query._validate(rel['subquery'])
else:
model = rel['model']
method = rel['method']
if method is None:
model_registry.get_manager(model).model_class
else:
model_registry.get_manager(model).getattr(method)
def get(self, request, model_name, id):
try:
cls = model_registry.get_manager(model_name).model_class
except:
return self._error(404, "Unknown model '%s'" % model_name)
try:
obj = cls.objects.get(pk=id)
except:
return self._error(
404, "Cannot find instance '%s' on '%s'" % (id, model_name))
return self._return(200, ObjectView.object_to_dict(obj))
def _validate(query):
"""
Validate a query. A query is an array whose elements are model
relationships or subqueries. This function checks each model relationship
to make sure the model and the relationship exist.
"""
for rel in query:
if 'orquery' in rel:
for q in rel['orquery']:
Query._validate(q)
elif 'subquery' in rel:
Query._validate(rel['subquery'])
else:
model = rel['model']
method = rel['method']
if method is None:
model_registry.get_manager(model).model_class
else:
model_registry.get_manager(model).getattr(method)
def setUp(self):
blog = Blog(name='Databases')
blog.save()
self.blog = blog
# add custom rel
Blog.blog_to_my_models_ = blog_to_my_models_
# register model
model_registry.register(Blog)
model_registry.register(MyModel)
model_registry.get_manager('Blog').allowed_relationships = ['blog_to_my_models_']
def _process(self, params):
if 'q' not in params:
return self._error(400, 'Missing query')
q = params['q']
check_only = get_param_value(params, 'c', False)
load_data = get_param_value(params, 'd', False)
follow_fk = get_param_value(params, 'fk', True)
ignore_excludes = get_param_value(params, 'x', False)
force = get_param_value(params, 'r', False)
force_cache = get_param_value(params, 'fc', False)
app = params['app'] if 'app' in params else None
try:
query = Query(q)
except Exception as e:
import traceback
traceback.print_exc()
return self._error(400, str(e))
# check query mode
if check_only:
return self._return(200, dict(query=q))
try:
results = self.get_query_results(query, force, force_cache, app)
except Exception as e:
import traceback
traceback.print_exc()
return self._error(400, str(e))
t = datetime.now()-results['computed_on']
if t.seconds > 300:
results['computed_since'] = str(naturaltime(results['computed_on']))
results['computed_on'] = str(results['computed_on'])
# data mode
objects = []
if load_data:
for result in results['results']:
if result['model']:
model = model_registry.get_manager(result['model']).model_class
ids = [t[0] for t in result['objects']]
objs = ModelView.get_objects_as_json(model, ids, ignore_excludes, follow_fk, force, app)
objects.append(objs)
else:
objects.append([])
results['data'] = objects
# print results
return self._return(200, results)
def setUp(self):
blog = Blog(name='Databases')
blog.save()
self.blog = blog
headlines = ['MySQL is a relational DB']*TestBatchFetch.N
self.entries = [Entry(headline=headline, blog=blog) for i, headline in enumerate(headlines)]
for entry in self.entries:
entry.save()
# register model
model_registry.register(curious_tests.models)
model_registry.get_manager('Blog').allowed_relationships = ['authors']
def setUp(self):
self.blog = Blog(name='Databases')
self.blog.save()
self.entries = [Entry(headline='Abc {}'.format(i), blog=self.blog) for i in xrange(self.N)]
for index, entry in enumerate(self.entries):
if index < len(self.entries) - 1:
entry.related_blog_id = self.entries[index + 1].pk
for entry in self.entries:
entry.save()
self.query_count = len(connection.queries)
model_registry.register(curious_tests.models)
Blog.entry_ = remote_fk('entry_id', Entry)
Entry.related_blog_ = remote_fk('related_blog_id', Entry)
model_registry.get_manager('Entry').allowed_relationships = [
'related_blog_',
]
def _graph_step(obj_src, model, step_f, filters, tree=None):
"""
Traverse one step on the graph. Takes in and returns arrays of output,
input object tuples. The input objects in the tuples are from start of the
query, not start of this step.
"""
# check if type matches existing object type
if len(obj_src):
t = type(obj_src[0][0])
if hasattr(t, '_deferred') and t._deferred:
t = t.__base__
if t != model_registry.get_manager(model).model_class:
raise Exception('Type mismatch when executing query: expecting "%s", got "%s"' %
(model, type(obj_src[0][0])))
next_obj_src = traverse([obj for obj, src in obj_src], step_f, filters)
if tree is not None:
tree.extend((t[0].id, t[1]) for t in next_obj_src)
return Query._extend_result(obj_src, next_obj_src)
def _rel_step(obj_src, step):
"""
Traverse a relationship, possibly recursively. Takes in and returns arrays
of output, input object tuples. The input objects in the tuples are from
start of the query, not start of this step.
"""
tree = None
if 'recursive' not in step or step['recursive'] is False:
model = step['model']
method = step['method']
filters = step['filters']
step_f = model_registry.get_manager(model).getattr(method)
obj_src = Query._graph_step(obj_src, model, step_f, filters)
else:
obj_src, tree = Query._recursive_rel(obj_src, step)
# print '%s: %d' % (step, len(obj_src))
return obj_src, tree
def _rel_step(obj_src, step):
"""
Traverse a relationship, possibly recursively. Takes in and returns arrays
of output, input object tuples. The input objects in the tuples are from
start of the query, not start of this step.
"""
tree = None
if 'recursive' not in step or step['recursive'] is False:
model = step['model']
method = step['method']
filters = step['filters']
step_f = model_registry.get_manager(model).getattr(method)
obj_src = Query._graph_step(obj_src, model, step_f, filters)
else:
obj_src, tree = Query._recursive_rel(obj_src, step)
# print '%s: %d' % (step, len(obj_src))
return obj_src, tree
def test_property_fields(self):
person_manager = model_registry.get_manager('Person')
person_manager.property_fields = [
'example_property_field',
]
r = self.client.post(
'/curious/q/',
data=json.dumps({'q': 'Person({0.pk})'.format(self.person), 'd': 1}),
content_type='application/json',
)
self.assertEquals(r.status_code, 200)
result = json.loads(r.content)['result']
print result
data = json.loads(r.content)['result']['data'][0]
self.assertItemsEqual(data['fields'], ['id', 'alive', 'example_property_field', 'gender'])
self.assertItemsEqual(data['objects'][0], [
self.person.pk,
self.person.alive,
self.person.example_property_field,
self.person.gender,
])
def _graph_step(obj_src, model, step_f, filters, tree=None):
"""
Traverse one step on the graph. Takes in and returns arrays of output,
input object tuples. The input objects in the tuples are from start of the
query, not start of this step.
"""
# check if type matches existing object type
if len(obj_src):
t = type(obj_src[0][0])
if hasattr(t, '_deferred') and t._deferred:
t = t.__base__
if t != model_registry.get_manager(model).model_class:
raise Exception(
'Type mismatch when executing query: expecting "%s", got "%s"'
% (model, type(obj_src[0][0])))
next_obj_src = traverse([obj for obj, src in obj_src], step_f, filters)
if tree is not None:
tree.extend((t[0].id, t[1]) for t in next_obj_src)
return Query._extend_result(obj_src, next_obj_src)
def setUp(self):
self.blog = Blog(name='Databases')
self.blog.save()
self.entries = [
Entry(headline='Abc {}'.format(i), blog=self.blog)
for i in xrange(self.N)
]
for index, entry in enumerate(self.entries):
if index < len(self.entries) - 1:
entry.related_blog_id = self.entries[index + 1].pk
for entry in self.entries:
entry.save()
self.query_count = len(connection.queries)
model_registry.register(curious_tests.models)
Blog.entry_ = remote_fk('entry_id', Entry)
Entry.related_blog_ = remote_fk('related_blog_id', Entry)
model_registry.get_manager('Entry').allowed_relationships = [
'related_blog_',
]
def get(self, request, model_name):
try:
cls = model_registry.get_manager(model_name).model_class
except Exception as e:
return self._error(404, "Unknown model '%s': %s" % (model_name, str(e)))
return self._return(200, ModelView.model_to_dict(cls))
def objects_to_dict(objects, ignore_excludes=False, follow_fk=True):
if len(objects) == 0:
return dict(fields=[], objects=[])
fields = []
fk = []
is_model = True
add_pk = False
obj = objects[0]
model_name = model_registry.get_name(obj.__class__)
if ignore_excludes is True:
excludes = []
else:
excludes = model_registry.get_manager(model_name).field_excludes
if hasattr(obj, '_meta'):
for f in obj._meta.fields:
if f.column not in excludes:
fields.append(f.column)
fk.append(f.name if type(f) == ForeignKey else None)
if 'id' not in fields:
fields.append('pk')
fk.append(None)
add_pk = True
else:
is_model = False
for f in obj.fields():
fields.append(f)
fk.append(None)
packed = []
urls = []
for obj in objects:
obj_url = model_registry.get_manager(model_name).url_of(obj)
values = []
for column, fk_name in zip(fields, fk):
if is_model:
value = getattr(obj, column)
else:
value = obj.get(column)
if type(value) is Decimal:
value = float(value)
elif not type(value) in (long, int, float, bool, types.NoneType):
value = unicode(value)
if fk_name is not None and follow_fk is True:
v = getattr(obj, fk_name)
if v is not None:
fk_model_name = model_registry.get_name(v.__class__)
fk_url = None
try:
fk_url = model_registry.get_manager(fk_model_name).url_of(v)
except:
fk_url = None
value = (fk_model_name, v.pk, str(v), fk_url)
values.append(value)
urls.append(obj_url)
packed.append(values)
if add_pk is True:
assert fields[-1] == 'pk'
fields[-1] = 'id'
return dict(fields=fields, objects=packed, urls=urls)
def _recursive_rel(obj_src, step):
"""
Traverse a relationship recursively. Collected objects, either loop
terminating objects or loop continuing objects. Returns arrays of output,
input object tuples. The input objects in the tuples are from start of the
query, not start of this step.
"""
model = step['model']
method = step['method']
filters = step['filters']
collect = step['collect']
step_f = model_registry.get_manager(model).getattr(method)
collected = {}
tree = []
starting = True
if collect == 'search' and filters is None:
return obj_src
to_remove = []
if collect in ("all", "until", "search"):
# if traversal or search, then keep starting nodes if starting nodes pass filter
if filters in (None, {}, []):
for tup in obj_src:
collected[tup] = 1
else:
filter_f = mk_filter_function(filters)
if len(obj_src) > 0:
ids = [obj.id for obj, src in obj_src]
q = obj_src[0][0].__class__.objects.filter(id__in=ids)
q = filter_f(q)
matched_objs = {obj.pk: 1 for obj in q}
for tup in obj_src:
if tup[0].pk in matched_objs:
collected[tup] = 1
elif collect == 'until':
# cannot continue to search with this starting node
to_remove.append(tup)
if len(to_remove) > 0:
obj_src = [tup for tup in obj_src if tup not in to_remove]
visited = {}
while len(obj_src) > 0:
# prevent loops by removing previously encountered edges; because many
# edges can lead to the same object, preventing revisit of edges rather
# than objects avoids loops without missing out on an edge.
new_src = [tup for tup in obj_src if tup not in visited]
for tup in obj_src:
visited[tup] = 1
if len(new_src) == 0:
break
next_obj_src = Query._graph_step(new_src, model, step_f, filters,
tree)
# print "from %s\nreach %s" % (new_src, next_obj_src)
if collect == 'terminal':
next_demux = Query._graph_step([(obj, obj.pk)
for obj, src in obj_src],
model, step_f, filters)
next_src = [t[1] for t in next_demux]
for tup in obj_src:
if tup[0].pk not in next_src:
if tup not in collected:
collected[tup] = 1
obj_src = next_obj_src
elif collect == 'search':
reachable = Query._graph_step(obj_src, model, step_f, None)
for tup in next_obj_src:
if tup not in collected:
collected[tup] = 1
obj_src = list(set(reachable) - set(next_obj_src))
elif collect == 'until':
for tup in next_obj_src:
if tup not in collected:
collected[tup] = 1
obj_src = next_obj_src
else: # traversal
reachable = Query._graph_step(obj_src, model, step_f, None)
for tup in next_obj_src:
if tup not in collected:
collected[tup] = 1
obj_src = reachable
return collected.keys(), tree
def _recursive_rel(obj_src, step):
"""
Traverse a relationship recursively. Collected objects, either loop
terminating objects or loop continuing objects. Returns arrays of output,
input object tuples. The input objects in the tuples are from start of the
query, not start of this step.
"""
model = step['model']
method = step['method']
filters = step['filters']
collect = step['collect']
step_f = model_registry.get_manager(model).getattr(method)
collected = {}
tree = []
starting = True
if collect == 'search' and filters is None:
return obj_src
to_remove = []
if collect in ("all", "until", "search"):
# if traversal or search, then keep starting nodes if starting nodes pass filter
if filters in (None, {}, []):
for tup in obj_src:
collected[tup] = 1
else:
filter_f = mk_filter_function(filters)
if len(obj_src) > 0:
ids = [obj.id for obj, src in obj_src]
q = obj_src[0][0].__class__.objects.filter(id__in=ids)
q = filter_f(q)
matched_objs = {obj.pk: 1 for obj in q}
for tup in obj_src:
if tup[0].pk in matched_objs:
collected[tup] = 1
elif collect == 'until':
# cannot continue to search with this starting node
to_remove.append(tup)
if len(to_remove) > 0:
obj_src = [tup for tup in obj_src if tup not in to_remove]
visited = {}
while len(obj_src) > 0:
# prevent loops by removing previously visited objects
new_src = [tup for tup in obj_src if tup[0].pk not in visited]
for obj, src in obj_src:
visited[obj.pk] = 1
if len(new_src) == 0:
break
next_obj_src = Query._graph_step(new_src, model, step_f, filters, tree)
if collect == 'terminal':
next_demux = Query._graph_step([(obj, obj.pk) for obj, src in obj_src], model, step_f, filters)
next_src = [t[1] for t in next_demux]
for tup in obj_src:
if tup[0].pk not in next_src:
if tup not in collected:
collected[tup] = 1
obj_src = next_obj_src
elif collect == 'search':
reachable = Query._graph_step(obj_src, model, step_f, None)
for tup in next_obj_src:
if tup not in collected:
collected[tup] = 1
obj_src = list(set(reachable)-set(next_obj_src))
elif collect == 'until':
for tup in next_obj_src:
if tup not in collected:
collected[tup] = 1
obj_src = next_obj_src
else: # traversal
reachable = Query._graph_step(obj_src, model, step_f, None)
for tup in next_obj_src:
if tup not in collected:
collected[tup] = 1
obj_src = reachable
return collected.keys(), tree
def objects_to_dict(objects, ignore_excludes=False, follow_fk=True):
if len(objects) == 0:
return dict(fields=[], objects=[])
fields = []
fk = []
is_model = True
add_pk = False
obj = objects[0]
model_name = model_registry.get_name(obj.__class__)
if ignore_excludes is True:
excludes = []
else:
excludes = model_registry.get_manager(model_name).field_excludes
if hasattr(obj, '_meta'):
for f in obj._meta.fields:
if f.column not in excludes:
fields.append(f.column)
fk.append(f.name if type(f) == ForeignKey else None)
if 'id' not in fields:
fields.append('pk')
fk.append(None)
add_pk = True
else:
is_model = False
for f in obj.fields():
fields.append(f)
fk.append(None)
packed = []
urls = []
for obj in objects:
obj_url = model_registry.get_manager(model_name).url_of(obj)
values = []
for column, fk_name in zip(fields, fk):
if is_model:
value = getattr(obj, column)
else:
value = obj.get(column)
if type(value) is Decimal:
value = float(value)
elif not type(value) in (long, int, float, bool,
types.NoneType):
value = unicode(value)
if fk_name is not None and follow_fk is True:
v = getattr(obj, fk_name)
if v is not None:
fk_model_name = model_registry.get_name(v.__class__)
fk_url = None
try:
fk_url = model_registry.get_manager(
fk_model_name).url_of(v)
except:
fk_url = None
value = (fk_model_name, v.pk, str(v), fk_url)
values.append(value)
urls.append(obj_url)
packed.append(values)
if add_pk is True:
assert fields[-1] == 'pk'
fields[-1] = 'id'
return dict(fields=fields, objects=packed, urls=urls)
