def object_to_dict(obj): d = {} for f in obj._meta.fields: d[f.column] = getattr(obj, f.column) if not type(d[f.column]) in (long, int, float, bool, types.NoneType): d[f.column] = unicode(d[f.column]) if type(f) == ForeignKey: v = getattr(obj, f.name) if v is not None: model_name = model_registry.get_name(v.__class__) d[f.column] = { 'model': model_name, 'id': v.pk, '__str__': str(v) } try: d[f.column]['url'] = model_registry.get_manager(model_name).url_of(v) except: pass if 'id' not in d: pk = obj.pk if not type(pk) in (long, int, float, bool, types.NoneType): d['id'] = unicode(pk) else: d['id'] = pk return d
def object_to_dict(obj): d = {} for f in obj._meta.fields: d[f.column] = getattr(obj, f.column) if not type( d[f.column]) in (long, int, float, bool, types.NoneType): d[f.column] = unicode(d[f.column]) if type(f) == ForeignKey: v = getattr(obj, f.name) if v is not None: model_name = model_registry.get_name(v.__class__) d[f.column] = { 'model': model_name, 'id': v.pk, '__str__': str(v) } try: d[f.column]['url'] = model_registry.get_manager( model_name).url_of(v) except: pass if 'id' not in d: pk = obj.pk if not type(pk) in (long, int, float, bool, types.NoneType): d['id'] = unicode(pk) else: d['id'] = pk return d
def setUp(self): blog = Blog(name='Databases') blog.save() self.blogs = [blog] authors = ('John Smith', 'Jane Doe', 'Joe Plummer') headlines = ('MySQL is a relational DB', 'Postgres is a really good relational DB', 'Neo4J is a graph DB') self.entries = [ Entry(headline=headline, blog=blog) for headline in headlines ] for entry in self.entries: entry.save() self.authors = [Author(name=name) for name in authors] for author in self.authors: author.save() for i, entry in enumerate(self.entries): entry.authors.add(self.authors[i]) entry.authors.add(self.authors[(i + 1) % len(self.authors)]) model_registry.register(curious_tests.models) model_registry.get_manager('Blog').allowed_relationships = ['authors']
def get(self, request, model_name): try: cls = model_registry.get_manager(model_name).model_class except Exception as e: return self._error(404, "Unknown model '%s': %s" % (model_name, str(e))) return self._return(200, ModelView.model_to_dict(cls))
def setUp(self): names = ('Databases', 'Relational Databases', 'Graph Databases') authors = ('John Smith', 'Jane Doe', 'Joe Plummer') headlines = ('MySQL is a relational DB', 'Postgres is a really good relational DB', 'Neo4J is a graph DB') self.blogs = [Blog(name=name) for name in names] for blog in self.blogs: blog.save() self.entries = [Entry(headline=headline, blog=blog) for headline, blog in zip(headlines, self.blogs)] for entry in self.entries: entry.save() self.authors = [Author(name=name) for name in authors] for author in self.authors: author.save() for i, entry in enumerate(self.entries): entry.authors.add(self.authors[i]) entry.authors.add(self.authors[(i+1)%len(self.authors)]) model_registry.register(curious_tests.models) model_registry.get_manager('Blog').allowed_relationships = ['authors']
def post(self, request, model_name): """ Fetch objects in batch. """ try: data = json.loads(request.body) except: return self._error(400, "Bad data") if 'ids' not in data: return self._error(400, "Missing ids array") app = None if 'app' in data: app = data['app'] try: cls = model_registry.get_manager(model_name).model_class except: return self._error(404, "Unknown model '%s'" % model_name) ignore_excludes = get_param_value(data, 'x', False) r = ModelView.get_objects_as_json(cls, data['ids'], ignore_excludes, True, True, app) return self._return(200, r)
def model_to_dict(cls): model_name = model_registry.get_name(cls) d = dict(model=model_name, relationships=[]) for f in dir(cls): if model_registry.get_manager(model_name).is_rel_allowed(f): d['relationships'].append(f) return d
def test_property_fields(self): person_manager = model_registry.get_manager('Person') person_manager.property_fields = [ 'example_property_field', ] r = self.client.post( '/curious/q/', data=json.dumps({ 'q': 'Person({0.pk})'.format(self.person), 'd': 1 }), content_type='application/json', ) self.assertEquals(r.status_code, 200) result = json.loads(r.content)['result'] print result data = json.loads(r.content)['result']['data'][0] self.assertItemsEqual( data['fields'], ['id', 'alive', 'example_property_field', 'gender']) self.assertItemsEqual(data['objects'][0], [ self.person.pk, self.person.alive, self.person.example_property_field, self.person.gender, ])
def __get_objects(self): """ Get initial objects from object query. """ model = self.__obj_query['model'] method = self.__obj_query['method'] filters = self.__obj_query['filters'] filter_f = mk_filter_function(filters) if method is None: cls = model_registry.get_manager(model).model_class q = cls.objects.all() q = filter_f(q) return q else: f = model_registry.get_manager(model).getattr(method) return f(filter_f)
def get(self, request, model_name, id): try: cls = model_registry.get_manager(model_name).model_class except: return self._error(404, "Unknown model '%s'" % model_name) try: obj = cls.objects.get(pk=id) except: return self._error(404, "Cannot find instance '%s' on '%s'" % (id, model_name)) return self._return(200, ObjectView.object_to_dict(obj))
def _process(self, params): if 'q' not in params: return self._error(400, 'Missing query') q = params['q'] check_only = get_param_value(params, 'c', False) load_data = get_param_value(params, 'd', False) follow_fk = get_param_value(params, 'fk', True) ignore_excludes = get_param_value(params, 'x', False) force = get_param_value(params, 'r', False) force_cache = get_param_value(params, 'fc', False) app = params['app'] if 'app' in params else None try: query = Query(q) except Exception as e: import traceback traceback.print_exc() return self._error(400, str(e)) # check query mode if check_only: return self._return(200, dict(query=q)) try: results = self.get_query_results(query, force, force_cache, app) except Exception as e: import traceback traceback.print_exc() return self._error(400, str(e)) t = datetime.now() - results['computed_on'] if t.seconds > 300: results['computed_since'] = str(naturaltime( results['computed_on'])) results['computed_on'] = str(results['computed_on']) # data mode objects = [] if load_data: for result in results['results']: if result['model']: model = model_registry.get_manager( result['model']).model_class ids = [t[0] for t in result['objects']] objs = ModelView.get_objects_as_json( model, ids, ignore_excludes, follow_fk, force, app) objects.append(objs) else: objects.append([]) results['data'] = objects # print results return self._return(200, results)
def _validate(query): """ Validate a query. A query is an array whose elements are model relationships or subqueries. This function checks each model relationship to make sure the model and the relationship exist. """ for rel in query: if 'orquery' in rel: for q in rel['orquery']: Query._validate(q) elif 'subquery' in rel: Query._validate(rel['subquery']) else: model = rel['model'] method = rel['method'] if method is None: model_registry.get_manager(model).model_class else: model_registry.get_manager(model).getattr(method)
def get(self, request, model_name, id): try: cls = model_registry.get_manager(model_name).model_class except: return self._error(404, "Unknown model '%s'" % model_name) try: obj = cls.objects.get(pk=id) except: return self._error( 404, "Cannot find instance '%s' on '%s'" % (id, model_name)) return self._return(200, ObjectView.object_to_dict(obj))
def setUp(self): blog = Blog(name='Databases') blog.save() self.blog = blog # add custom rel Blog.blog_to_my_models_ = blog_to_my_models_ # register model model_registry.register(Blog) model_registry.register(MyModel) model_registry.get_manager('Blog').allowed_relationships = ['blog_to_my_models_']
def _process(self, params): if 'q' not in params: return self._error(400, 'Missing query') q = params['q'] check_only = get_param_value(params, 'c', False) load_data = get_param_value(params, 'd', False) follow_fk = get_param_value(params, 'fk', True) ignore_excludes = get_param_value(params, 'x', False) force = get_param_value(params, 'r', False) force_cache = get_param_value(params, 'fc', False) app = params['app'] if 'app' in params else None try: query = Query(q) except Exception as e: import traceback traceback.print_exc() return self._error(400, str(e)) # check query mode if check_only: return self._return(200, dict(query=q)) try: results = self.get_query_results(query, force, force_cache, app) except Exception as e: import traceback traceback.print_exc() return self._error(400, str(e)) t = datetime.now()-results['computed_on'] if t.seconds > 300: results['computed_since'] = str(naturaltime(results['computed_on'])) results['computed_on'] = str(results['computed_on']) # data mode objects = [] if load_data: for result in results['results']: if result['model']: model = model_registry.get_manager(result['model']).model_class ids = [t[0] for t in result['objects']] objs = ModelView.get_objects_as_json(model, ids, ignore_excludes, follow_fk, force, app) objects.append(objs) else: objects.append([]) results['data'] = objects # print results return self._return(200, results)
def setUp(self): blog = Blog(name='Databases') blog.save() self.blog = blog headlines = ['MySQL is a relational DB']*TestBatchFetch.N self.entries = [Entry(headline=headline, blog=blog) for i, headline in enumerate(headlines)] for entry in self.entries: entry.save() # register model model_registry.register(curious_tests.models) model_registry.get_manager('Blog').allowed_relationships = ['authors']
def setUp(self): self.blog = Blog(name='Databases') self.blog.save() self.entries = [Entry(headline='Abc {}'.format(i), blog=self.blog) for i in xrange(self.N)] for index, entry in enumerate(self.entries): if index < len(self.entries) - 1: entry.related_blog_id = self.entries[index + 1].pk for entry in self.entries: entry.save() self.query_count = len(connection.queries) model_registry.register(curious_tests.models) Blog.entry_ = remote_fk('entry_id', Entry) Entry.related_blog_ = remote_fk('related_blog_id', Entry) model_registry.get_manager('Entry').allowed_relationships = [ 'related_blog_', ]
def _graph_step(obj_src, model, step_f, filters, tree=None): """ Traverse one step on the graph. Takes in and returns arrays of output, input object tuples. The input objects in the tuples are from start of the query, not start of this step. """ # check if type matches existing object type if len(obj_src): t = type(obj_src[0][0]) if hasattr(t, '_deferred') and t._deferred: t = t.__base__ if t != model_registry.get_manager(model).model_class: raise Exception('Type mismatch when executing query: expecting "%s", got "%s"' % (model, type(obj_src[0][0]))) next_obj_src = traverse([obj for obj, src in obj_src], step_f, filters) if tree is not None: tree.extend((t[0].id, t[1]) for t in next_obj_src) return Query._extend_result(obj_src, next_obj_src)
def _rel_step(obj_src, step): """ Traverse a relationship, possibly recursively. Takes in and returns arrays of output, input object tuples. The input objects in the tuples are from start of the query, not start of this step. """ tree = None if 'recursive' not in step or step['recursive'] is False: model = step['model'] method = step['method'] filters = step['filters'] step_f = model_registry.get_manager(model).getattr(method) obj_src = Query._graph_step(obj_src, model, step_f, filters) else: obj_src, tree = Query._recursive_rel(obj_src, step) # print '%s: %d' % (step, len(obj_src)) return obj_src, tree
def test_property_fields(self): person_manager = model_registry.get_manager('Person') person_manager.property_fields = [ 'example_property_field', ] r = self.client.post( '/curious/q/', data=json.dumps({'q': 'Person({0.pk})'.format(self.person), 'd': 1}), content_type='application/json', ) self.assertEquals(r.status_code, 200) result = json.loads(r.content)['result'] print result data = json.loads(r.content)['result']['data'][0] self.assertItemsEqual(data['fields'], ['id', 'alive', 'example_property_field', 'gender']) self.assertItemsEqual(data['objects'][0], [ self.person.pk, self.person.alive, self.person.example_property_field, self.person.gender, ])
def _graph_step(obj_src, model, step_f, filters, tree=None): """ Traverse one step on the graph. Takes in and returns arrays of output, input object tuples. The input objects in the tuples are from start of the query, not start of this step. """ # check if type matches existing object type if len(obj_src): t = type(obj_src[0][0]) if hasattr(t, '_deferred') and t._deferred: t = t.__base__ if t != model_registry.get_manager(model).model_class: raise Exception( 'Type mismatch when executing query: expecting "%s", got "%s"' % (model, type(obj_src[0][0]))) next_obj_src = traverse([obj for obj, src in obj_src], step_f, filters) if tree is not None: tree.extend((t[0].id, t[1]) for t in next_obj_src) return Query._extend_result(obj_src, next_obj_src)
def setUp(self): self.blog = Blog(name='Databases') self.blog.save() self.entries = [ Entry(headline='Abc {}'.format(i), blog=self.blog) for i in xrange(self.N) ] for index, entry in enumerate(self.entries): if index < len(self.entries) - 1: entry.related_blog_id = self.entries[index + 1].pk for entry in self.entries: entry.save() self.query_count = len(connection.queries) model_registry.register(curious_tests.models) Blog.entry_ = remote_fk('entry_id', Entry) Entry.related_blog_ = remote_fk('related_blog_id', Entry) model_registry.get_manager('Entry').allowed_relationships = [ 'related_blog_', ]
def objects_to_dict(objects, ignore_excludes=False, follow_fk=True): if len(objects) == 0: return dict(fields=[], objects=[]) fields = [] fk = [] is_model = True add_pk = False obj = objects[0] model_name = model_registry.get_name(obj.__class__) if ignore_excludes is True: excludes = [] else: excludes = model_registry.get_manager(model_name).field_excludes if hasattr(obj, '_meta'): for f in obj._meta.fields: if f.column not in excludes: fields.append(f.column) fk.append(f.name if type(f) == ForeignKey else None) if 'id' not in fields: fields.append('pk') fk.append(None) add_pk = True else: is_model = False for f in obj.fields(): fields.append(f) fk.append(None) packed = [] urls = [] for obj in objects: obj_url = model_registry.get_manager(model_name).url_of(obj) values = [] for column, fk_name in zip(fields, fk): if is_model: value = getattr(obj, column) else: value = obj.get(column) if type(value) is Decimal: value = float(value) elif not type(value) in (long, int, float, bool, types.NoneType): value = unicode(value) if fk_name is not None and follow_fk is True: v = getattr(obj, fk_name) if v is not None: fk_model_name = model_registry.get_name(v.__class__) fk_url = None try: fk_url = model_registry.get_manager(fk_model_name).url_of(v) except: fk_url = None value = (fk_model_name, v.pk, str(v), fk_url) values.append(value) urls.append(obj_url) packed.append(values) if add_pk is True: assert fields[-1] == 'pk' fields[-1] = 'id' return dict(fields=fields, objects=packed, urls=urls)
def _recursive_rel(obj_src, step): """ Traverse a relationship recursively. Collected objects, either loop terminating objects or loop continuing objects. Returns arrays of output, input object tuples. The input objects in the tuples are from start of the query, not start of this step. """ model = step['model'] method = step['method'] filters = step['filters'] collect = step['collect'] step_f = model_registry.get_manager(model).getattr(method) collected = {} tree = [] starting = True if collect == 'search' and filters is None: return obj_src to_remove = [] if collect in ("all", "until", "search"): # if traversal or search, then keep starting nodes if starting nodes pass filter if filters in (None, {}, []): for tup in obj_src: collected[tup] = 1 else: filter_f = mk_filter_function(filters) if len(obj_src) > 0: ids = [obj.id for obj, src in obj_src] q = obj_src[0][0].__class__.objects.filter(id__in=ids) q = filter_f(q) matched_objs = {obj.pk: 1 for obj in q} for tup in obj_src: if tup[0].pk in matched_objs: collected[tup] = 1 elif collect == 'until': # cannot continue to search with this starting node to_remove.append(tup) if len(to_remove) > 0: obj_src = [tup for tup in obj_src if tup not in to_remove] visited = {} while len(obj_src) > 0: # prevent loops by removing previously encountered edges; because many # edges can lead to the same object, preventing revisit of edges rather # than objects avoids loops without missing out on an edge. new_src = [tup for tup in obj_src if tup not in visited] for tup in obj_src: visited[tup] = 1 if len(new_src) == 0: break next_obj_src = Query._graph_step(new_src, model, step_f, filters, tree) # print "from %s\nreach %s" % (new_src, next_obj_src) if collect == 'terminal': next_demux = Query._graph_step([(obj, obj.pk) for obj, src in obj_src], model, step_f, filters) next_src = [t[1] for t in next_demux] for tup in obj_src: if tup[0].pk not in next_src: if tup not in collected: collected[tup] = 1 obj_src = next_obj_src elif collect == 'search': reachable = Query._graph_step(obj_src, model, step_f, None) for tup in next_obj_src: if tup not in collected: collected[tup] = 1 obj_src = list(set(reachable) - set(next_obj_src)) elif collect == 'until': for tup in next_obj_src: if tup not in collected: collected[tup] = 1 obj_src = next_obj_src else: # traversal reachable = Query._graph_step(obj_src, model, step_f, None) for tup in next_obj_src: if tup not in collected: collected[tup] = 1 obj_src = reachable return collected.keys(), tree
def _recursive_rel(obj_src, step): """ Traverse a relationship recursively. Collected objects, either loop terminating objects or loop continuing objects. Returns arrays of output, input object tuples. The input objects in the tuples are from start of the query, not start of this step. """ model = step['model'] method = step['method'] filters = step['filters'] collect = step['collect'] step_f = model_registry.get_manager(model).getattr(method) collected = {} tree = [] starting = True if collect == 'search' and filters is None: return obj_src to_remove = [] if collect in ("all", "until", "search"): # if traversal or search, then keep starting nodes if starting nodes pass filter if filters in (None, {}, []): for tup in obj_src: collected[tup] = 1 else: filter_f = mk_filter_function(filters) if len(obj_src) > 0: ids = [obj.id for obj, src in obj_src] q = obj_src[0][0].__class__.objects.filter(id__in=ids) q = filter_f(q) matched_objs = {obj.pk: 1 for obj in q} for tup in obj_src: if tup[0].pk in matched_objs: collected[tup] = 1 elif collect == 'until': # cannot continue to search with this starting node to_remove.append(tup) if len(to_remove) > 0: obj_src = [tup for tup in obj_src if tup not in to_remove] visited = {} while len(obj_src) > 0: # prevent loops by removing previously visited objects new_src = [tup for tup in obj_src if tup[0].pk not in visited] for obj, src in obj_src: visited[obj.pk] = 1 if len(new_src) == 0: break next_obj_src = Query._graph_step(new_src, model, step_f, filters, tree) if collect == 'terminal': next_demux = Query._graph_step([(obj, obj.pk) for obj, src in obj_src], model, step_f, filters) next_src = [t[1] for t in next_demux] for tup in obj_src: if tup[0].pk not in next_src: if tup not in collected: collected[tup] = 1 obj_src = next_obj_src elif collect == 'search': reachable = Query._graph_step(obj_src, model, step_f, None) for tup in next_obj_src: if tup not in collected: collected[tup] = 1 obj_src = list(set(reachable)-set(next_obj_src)) elif collect == 'until': for tup in next_obj_src: if tup not in collected: collected[tup] = 1 obj_src = next_obj_src else: # traversal reachable = Query._graph_step(obj_src, model, step_f, None) for tup in next_obj_src: if tup not in collected: collected[tup] = 1 obj_src = reachable return collected.keys(), tree
def objects_to_dict(objects, ignore_excludes=False, follow_fk=True): if len(objects) == 0: return dict(fields=[], objects=[]) fields = [] fk = [] is_model = True add_pk = False obj = objects[0] model_name = model_registry.get_name(obj.__class__) if ignore_excludes is True: excludes = [] else: excludes = model_registry.get_manager(model_name).field_excludes if hasattr(obj, '_meta'): for f in obj._meta.fields: if f.column not in excludes: fields.append(f.column) fk.append(f.name if type(f) == ForeignKey else None) if 'id' not in fields: fields.append('pk') fk.append(None) add_pk = True else: is_model = False for f in obj.fields(): fields.append(f) fk.append(None) packed = [] urls = [] for obj in objects: obj_url = model_registry.get_manager(model_name).url_of(obj) values = [] for column, fk_name in zip(fields, fk): if is_model: value = getattr(obj, column) else: value = obj.get(column) if type(value) is Decimal: value = float(value) elif not type(value) in (long, int, float, bool, types.NoneType): value = unicode(value) if fk_name is not None and follow_fk is True: v = getattr(obj, fk_name) if v is not None: fk_model_name = model_registry.get_name(v.__class__) fk_url = None try: fk_url = model_registry.get_manager( fk_model_name).url_of(v) except: fk_url = None value = (fk_model_name, v.pk, str(v), fk_url) values.append(value) urls.append(obj_url) packed.append(values) if add_pk is True: assert fields[-1] == 'pk' fields[-1] = 'id' return dict(fields=fields, objects=packed, urls=urls)