def test_getBuilds_resultSpecOrder(self): rs = resultspec.ResultSpec(order=['-started_at']) rs.fieldMapping = {'started_at': 'builds.started_at'} yield self.insertTestData(self.backgroundData + self.threeBuilds) bdicts = yield self.db.builds.getBuilds(resultSpec=rs) # applying the spec in the db layer should have emptied the order in # resultSpec self.assertEqual(rs.order, None) # assert applying the same order at the data layer will give the same # results rs = resultspec.ResultSpec(order=['-started_at']) ordered_bdicts = rs.apply(bdicts) self.assertEqual(ordered_bdicts, bdicts) # assert applying an opposite order at the data layer will give different # results rs = resultspec.ResultSpec(order=['started_at']) ordered_bdicts = rs.apply(bdicts) self.assertNotEqual(ordered_bdicts, bdicts)
def test_get_incomplete(self): f = resultspec.Filter('complete', 'eq', [False]) d = self.callGet(('buildsets',), resultSpec=resultspec.ResultSpec(filters=[f])) @d.addCallback def check(buildsets): self.assertEqual(len(buildsets), 1) self.validateData(buildsets[0]) self.assertEqual(buildsets[0]['bsid'], 14) return d
def testGetNoFilters(self): getBuildRequestsMock = mock.Mock(return_value={}) self.patch(self.master.db.buildrequests, 'getBuildRequests', getBuildRequestsMock) yield self.callGet(('buildrequests', )) getBuildRequestsMock.assert_called_with( builderid=None, bsid=None, complete=None, claimed=None, resultSpec=resultspec.ResultSpec())
def test_sort_null_datetimefields(self): data = mklist(('fn', 'ln'), ('albert', datetime.datetime(1, 1, 1)), ('cedric', None)) exp = mklist(('fn', 'ln'), ('cedric', None), ('albert', datetime.datetime(1, 1, 1))) self.assertListResultEqual( resultspec.ResultSpec(order=['ln']).apply(data), base.ListResult(exp, total=2))
def test_get_contains_two_tags_one_unknown(self): resultSpec = resultspec.ResultSpec( filters=[resultspec.Filter('tags', 'contains', ["tagA", "tagC"])]) builders = yield self.callGet(('builders',)) builders = resultSpec.apply(builders) for b in builders: self.validateData(b) self.assertEqual(sorted([b['builderid'] for b in builders]), [3, 5])
def test_pagination_prepaginated(self): data = base.ListResult(mklist('x', *lrange(10, 20))) data.offset = 10 data.total = 30 data.limit = 10 self.assertListResultEqual( # ResultSpec has its offset/limit fields cleared resultspec.ResultSpec().apply(data), base.ListResult(mklist('x', *lrange(10, 20)), offset=10, total=30, limit=10))
def test_getBuilds_resultSpecFilterContainsTwoValues(self): rs = resultspec.ResultSpec(filters=[ resultspec.Filter('state_string', 'contains', ['build 5', 'build 6']) ]) rs.fieldMapping = {'state_string': 'builds.state_string'} yield self.insertTestData(self.backgroundData + self.threeBuilds) bdicts = yield self.db.builds.getBuilds(resultSpec=rs) for bdict in bdicts: validation.verifyDbDict(self, 'dbbuilddict', bdict) self.assertEqual(sorted(bdicts, key=lambda bd: bd['id']), [self.threeBdicts[50], self.threeBdicts[51]])
def callGet(self, path, resultSpec=None): self.assertIsInstance(path, tuple) if resultSpec is None: resultSpec = resultspec.ResultSpec() endpoint, kwargs = self.matcher[path] self.assertIdentical(endpoint, self.ep) rv = yield endpoint.get(resultSpec, kwargs) if self.ep.isCollection: self.assertIsInstance(rv, (list, base.ListResult)) else: self.assertIsInstance(rv, (dict, type(None))) return rv
def get(self, path, filters=None, fields=None, order=None, limit=None, offset=None): resultSpec = resultspec.ResultSpec(filters=filters, fields=fields, order=order, limit=limit, offset=offset) return self.get_with_resultspec(path, resultSpec)
def test_get_contains_two_tags_one_unknown(self): resultSpec = resultspec.ResultSpec( filters=[resultspec.Filter('tags', 'contains', ["tagA", "tagC"])]) d = self.callGet(('builders', )) @d.addCallback def check(builders): builders = resultSpec.apply(builders) [self.validateData(b) for b in builders] self.assertEqual(sorted([b['builderid'] for b in builders]), [3, 5]) return d
def test_apply_ordering_multi(self): data = mklist(('fn', 'ln'), ('cedric', 'willis'), ('albert', 'engelbert'), ('bruce', 'willis'), ('dwayne', 'montague')) exp = base.ListResult(mklist(('fn', 'ln'), ('albert', 'engelbert'), ('dwayne', 'montague'), ('bruce', 'willis'), ('cedric', 'willis')), total=4) random.shuffle(data) self.assertListResultEqual( resultspec.ResultSpec(order=['ln', 'fn']).apply(data), exp) exp = base.ListResult(mklist(('fn', 'ln'), ('cedric', 'willis'), ('bruce', 'willis'), ('dwayne', 'montague'), ('albert', 'engelbert')), total=4) self.assertListResultEqual( resultspec.ResultSpec(order=['-ln', '-fn']).apply(data), exp)
def testGetProperties(self): self.master.db.insertTestData([ fakedb.BuildsetProperty(buildsetid=8822, property_name='prop1', property_value='["one", "fake1"]'), fakedb.BuildsetProperty(buildsetid=8822, property_name='prop2', property_value='["two", "fake2"]'), ]) prop = resultspec.Property(b'property', 'eq', '*') buildrequests = yield self.callGet(('builders', 78, 'buildrequests'), resultSpec=resultspec.ResultSpec(properties=[prop])) self.assertEqual(len(buildrequests), 1) self.assertEqual(buildrequests[0]['buildrequestid'], 46) self.assertEqual(buildrequests[0]['properties'], {u'prop1': (u'one', u'fake1'), u'prop2': (u'two', u'fake2')})
def testGetClaimedByMasterIdFilters(self): getBuildRequestsMock = mock.Mock(return_value={}) self.patch(self.master.db.buildrequests, 'getBuildRequests', getBuildRequestsMock) f1 = resultspec.Filter('claimed', 'eq', [True]) f2 = resultspec.Filter('claimed_by_masterid', 'eq', [fakedb.FakeBuildRequestsComponent.MASTER_ID]) yield self.callGet(('buildrequests', ), resultSpec=resultspec.ResultSpec(filters=[f1, f2])) getBuildRequestsMock.assert_called_with( builderid=None, bsid=None, complete=None, claimed=fakedb.FakeBuildRequestsComponent.MASTER_ID)
def do_test_pagination(self, bareList): data = mklist('x', *lrange(101, 131)) if not bareList: data = base.ListResult(data) data.offset = None data.total = len(data) data.limit = None self.assertListResultEqual( resultspec.ResultSpec(offset=0).apply(data), base.ListResult(mklist('x', *lrange(101, 131)), offset=0, total=30)) self.assertListResultEqual( resultspec.ResultSpec(offset=10).apply(data), base.ListResult(mklist('x', *lrange(111, 131)), offset=10, total=30)) self.assertListResultEqual( resultspec.ResultSpec(offset=10, limit=10).apply(data), base.ListResult(mklist('x', *lrange(111, 121)), offset=10, total=30, limit=10)) self.assertListResultEqual( resultspec.ResultSpec(offset=20, limit=15).apply(data), base.ListResult(mklist('x', *lrange(121, 131)), offset=20, total=30, limit=15)) # off the end
def testGetFilters(self): getBuildRequestsMock = mock.Mock(return_value={}) self.patch(self.master.db.buildrequests, 'getBuildRequests', getBuildRequestsMock) f1 = resultspec.Filter('complete', 'eq', [False]) f2 = resultspec.Filter('claimed', 'eq', [True]) f3 = resultspec.Filter('buildsetid', 'eq', [55]) f4 = resultspec.Filter('branch', 'eq', ['mybranch']) f5 = resultspec.Filter('repository', 'eq', ['myrepo']) yield self.callGet( ('buildrequests', ), resultSpec=resultspec.ResultSpec(filters=[f1, f2, f3, f4, f5])) getBuildRequestsMock.assert_called_with(builderid=None, bsid=55, complete=False, claimed=True)
def get(self, path, filters=None, fields=None, order=None, limit=None, offset=None): resultSpec = resultspec.ResultSpec(filters=filters, fields=fields, order=order, limit=limit, offset=offset) endpoint, kwargs = self.getEndpoint(path) rv = yield endpoint.get(resultSpec, kwargs) if resultSpec: rv = resultSpec.apply(rv) defer.returnValue(rv)
def callGet(self, path, resultSpec=None): self.assertIsInstance(path, tuple) if resultSpec is None: resultSpec = resultspec.ResultSpec() endpoint, kwargs = self.matcher[path] self.assertIdentical(endpoint, self.ep) d = endpoint.get(resultSpec, kwargs) self.assertIsInstance(d, defer.Deferred) @d.addCallback def checkNumber(rv): if self.ep.isCollection: self.assertIsInstance(rv, (list, base.ListResult)) else: self.assertIsInstance(rv, (dict, type(None))) return rv return d
def test_getChanges_missing(self): yield self.insertTestData(self.change13_rows + self.change14_rows) def check(changes): # requested all, but only got 2 # sort by changeid, since we assert on change 13 at index 0 changes.sort(key=lambda c: c['changeid']) changeids = [c['changeid'] for c in changes] self.assertEqual(changeids, [13, 14]) # double-check that they have .files, etc. self.assertEqual(sorted(changes[0]['files']), sorted(['master/README.txt', 'worker/README.txt'])) self.assertEqual(changes[0]['properties'], {'notest': ('no', 'Change')}) rs = resultspec.ResultSpec(order=['-changeid'], limit=5) changes = yield self.db.changes.getChanges(resultSpec=rs) check(changes) changes = yield self.db.changes.getChanges() check(changes)
def decodeResultSpec(self, request, endpoint): reqArgs = request.args def checkFields(fields, negOk=False): for k in fields: if k[0] == '-' and negOk: k = k[1:] if k not in entityType.fieldNames: raise BadRequest("no such field %r" % (k, )) entityType = endpoint.rtype.entityType limit = offset = order = fields = None filters, properties = [], [] for arg in reqArgs: if arg == 'order': order = reqArgs[arg] checkFields(order, True) continue elif arg == 'field': fields = reqArgs[arg] checkFields(fields, False) continue elif arg == 'limit': try: limit = int(reqArgs[arg][0]) except Exception: raise BadRequest('invalid limit') continue elif arg == 'offset': try: offset = int(reqArgs[arg][0]) except Exception: raise BadRequest('invalid offset') continue elif arg == 'property': try: props = [v.decode('utf-8') for v in reqArgs[arg]] except Exception: raise BadRequest('invalid property value for %s' % arg) properties.append(resultspec.Property(arg, 'eq', props)) continue elif arg in entityType.fieldNames: field = entityType.fields[arg] try: values = [field.valueFromString(v) for v in reqArgs[arg]] except Exception: raise BadRequest('invalid filter value for %s' % arg) filters.append(resultspec.Filter(arg, 'eq', values)) continue elif '__' in arg: field, op = arg.rsplit('__', 1) args = reqArgs[arg] operators = (resultspec.Filter.singular_operators if len(args) == 1 else resultspec.Filter.plural_operators) if op in operators and field in entityType.fieldNames: fieldType = entityType.fields[field] try: values = [ fieldType.valueFromString(v) for v in reqArgs[arg] ] except Exception: raise BadRequest('invalid filter value for %s' % arg) filters.append(resultspec.Filter(field, op, values)) continue raise BadRequest("unrecognized query parameter '%s'" % (arg, )) # if ordering or filtering is on a field that's not in fields, bail out if fields: fieldsSet = set(fields) if order and set(order) - fieldsSet: raise BadRequest("cannot order on un-selected fields") for filter in filters: if filter.field not in fieldsSet: raise BadRequest("cannot filter on un-selected fields") # build the result spec rspec = resultspec.ResultSpec(fields=fields, limit=limit, offset=offset, order=order, filters=filters, properties=properties) # for singular endpoints, only allow fields if not endpoint.isCollection: if rspec.filters: raise BadRequest("this is not a collection") return rspec
def resultspec_from_jsonapi(self, req_args, entityType, is_collection): def checkFields(fields, negOk=False): for field in fields: k = bytes2unicode(field) if k[0] == '-' and negOk: k = k[1:] if k not in entityType.fieldNames: raise exceptions.InvalidQueryParameter( "no such field '{}'".format(k)) limit = offset = order = fields = None filters, properties = [], [] limit = offset = order = fields = None filters, properties = [], [] for arg in req_args: argStr = bytes2unicode(arg) if argStr == 'order': order = tuple([bytes2unicode(o) for o in req_args[arg]]) checkFields(order, True) elif argStr == 'field': fields = req_args[arg] checkFields(fields, False) elif argStr == 'limit': try: limit = int(req_args[arg][0]) except Exception as e: raise exceptions.InvalidQueryParameter( 'invalid limit') from e elif argStr == 'offset': try: offset = int(req_args[arg][0]) except Exception as e: raise exceptions.InvalidQueryParameter( 'invalid offset') from e elif argStr == 'property': try: props = [] for v in req_args[arg]: if not isinstance(v, (bytes, str)): raise TypeError("Invalid type {} for {}".format( type(v), v)) props.append(bytes2unicode(v)) except Exception as e: raise exceptions.InvalidQueryParameter( 'invalid property value for {}'.format(arg)) from e properties.append(resultspec.Property(arg, 'eq', props)) elif argStr in entityType.fieldNames: field = entityType.fields[argStr] try: values = [field.valueFromString(v) for v in req_args[arg]] except Exception as e: raise exceptions.InvalidQueryParameter( 'invalid filter value for {}'.format(argStr)) from e filters.append(resultspec.Filter(argStr, 'eq', values)) elif '__' in argStr: field, op = argStr.rsplit('__', 1) args = req_args[arg] operators = (resultspec.Filter.singular_operators if len(args) == 1 else resultspec.Filter.plural_operators) if op in operators and field in entityType.fieldNames: fieldType = entityType.fields[field] try: values = [ fieldType.valueFromString(v) for v in req_args[arg] ] except Exception as e: raise exceptions.InvalidQueryParameter( 'invalid filter value for {}'.format( argStr)) from e filters.append(resultspec.Filter(field, op, values)) else: raise exceptions.InvalidQueryParameter( "unrecognized query parameter '{}'".format(argStr)) # if ordering or filtering is on a field that's not in fields, bail out if fields: fields = [bytes2unicode(f) for f in fields] fieldsSet = set(fields) if order and {o.lstrip('-') for o in order} - fieldsSet: raise exceptions.InvalidQueryParameter( "cannot order on un-selected fields") for filter in filters: if filter.field not in fieldsSet: raise exceptions.InvalidQueryParameter( "cannot filter on un-selected fields") # build the result spec rspec = resultspec.ResultSpec(fields=fields, limit=limit, offset=offset, order=order, filters=filters, properties=properties) # for singular endpoints, only allow fields if not is_collection: if rspec.filters: raise exceptions.InvalidQueryParameter( "this is not a collection") return rspec
def test_getChangesOtherOrder(self): resultSpec = resultspec.ResultSpec(limit=1, order=('-when_time_stamp',)) changes = yield self.callGet(('changes',), resultSpec=resultSpec) self.assertEqual(len(changes), 1)
def test_apply_None(self): self.assertEqual(resultspec.ResultSpec().apply(None), None)
def test_getChangesOtherOffset(self): resultSpec = resultspec.ResultSpec( limit=1, offset=1, order=('-changeid',)) changes = yield self.callGet(('changes',), resultSpec=resultSpec) self.assertEqual(len(changes), 1)
def test_popField(self): rs = resultspec.ResultSpec(fields=['foo', 'bar']) self.assertTrue(rs.popField('foo')) self.assertEqual(rs.fields, ['bar'])
def test_popField_not_present(self): rs = resultspec.ResultSpec(fields=['foo', 'bar']) self.assertFalse(rs.popField('nosuch')) self.assertEqual(rs.fields, ['foo', 'bar'])
def test_removeOrder(self): rs = resultspec.ResultSpec(order=['foo', '-bar']) rs.removeOrder() self.assertEqual(rs.order, None)
def test_popIntegerFilterNotInt(self): rs = resultspec.ResultSpec(filters=[ resultspec.Filter('foo', 'eq', ['bar']), ]) with self.assertRaises(ValueError): rs.popIntegerFilter('foo')
def test_popIntegerFilterSeveral(self): rs = resultspec.ResultSpec(filters=[ resultspec.Filter('foo', 'eq', ['12', '13']), ]) self.assertEqual(rs.popIntegerFilter('foo'), None)
def test_popStringFilterSeveral(self): rs = resultspec.ResultSpec(filters=[ resultspec.Filter('foo', 'eq', ['foo', 'bar']), ]) self.assertEqual(rs.popStringFilter('foo'), None)
def test_getChangesOtherOrder(self): resultSpec = resultspec.ResultSpec(limit=1, order=['-when_time_stamp']) changes = yield self.callGet(('changes',), resultSpec=resultSpec) # limit not implemented for other order self.assertEqual(len(changes), 2)