def test_getBuilds_resultSpecOrder(self):
rs = resultspec.ResultSpec(order=['-started_at'])
rs.fieldMapping = {'started_at': 'builds.started_at'}
yield self.insertTestData(self.backgroundData + self.threeBuilds)
bdicts = yield self.db.builds.getBuilds(resultSpec=rs)
# applying the spec in the db layer should have emptied the order in
# resultSpec
self.assertEqual(rs.order, None)
# assert applying the same order at the data layer will give the same
# results
rs = resultspec.ResultSpec(order=['-started_at'])
ordered_bdicts = rs.apply(bdicts)
self.assertEqual(ordered_bdicts, bdicts)
# assert applying an opposite order at the data layer will give different
# results
rs = resultspec.ResultSpec(order=['started_at'])
ordered_bdicts = rs.apply(bdicts)
self.assertNotEqual(ordered_bdicts, bdicts)
def test_get_incomplete(self):
f = resultspec.Filter('complete', 'eq', [False])
d = self.callGet(('buildsets',),
resultSpec=resultspec.ResultSpec(filters=[f]))
@d.addCallback
def check(buildsets):
self.assertEqual(len(buildsets), 1)
self.validateData(buildsets[0])
self.assertEqual(buildsets[0]['bsid'], 14)
return d
def testGetNoFilters(self):
getBuildRequestsMock = mock.Mock(return_value={})
self.patch(self.master.db.buildrequests, 'getBuildRequests',
getBuildRequestsMock)
yield self.callGet(('buildrequests', ))
getBuildRequestsMock.assert_called_with(
builderid=None,
bsid=None,
complete=None,
claimed=None,
resultSpec=resultspec.ResultSpec())
def test_sort_null_datetimefields(self):
data = mklist(('fn', 'ln'),
('albert', datetime.datetime(1, 1, 1)),
('cedric', None))
exp = mklist(('fn', 'ln'),
('cedric', None),
('albert', datetime.datetime(1, 1, 1)))
self.assertListResultEqual(
resultspec.ResultSpec(order=['ln']).apply(data),
base.ListResult(exp, total=2))
def test_get_contains_two_tags_one_unknown(self):
resultSpec = resultspec.ResultSpec(
filters=[resultspec.Filter('tags', 'contains', ["tagA", "tagC"])])
builders = yield self.callGet(('builders',))
builders = resultSpec.apply(builders)
for b in builders:
self.validateData(b)
self.assertEqual(sorted([b['builderid'] for b in builders]),
[3, 5])
def test_pagination_prepaginated(self):
data = base.ListResult(mklist('x', *lrange(10, 20)))
data.offset = 10
data.total = 30
data.limit = 10
self.assertListResultEqual(
# ResultSpec has its offset/limit fields cleared
resultspec.ResultSpec().apply(data),
base.ListResult(mklist('x', *lrange(10, 20)),
offset=10,
total=30,
limit=10))
def test_getBuilds_resultSpecFilterContainsTwoValues(self):
rs = resultspec.ResultSpec(filters=[
resultspec.Filter('state_string', 'contains',
['build 5', 'build 6'])
])
rs.fieldMapping = {'state_string': 'builds.state_string'}
yield self.insertTestData(self.backgroundData + self.threeBuilds)
bdicts = yield self.db.builds.getBuilds(resultSpec=rs)
for bdict in bdicts:
validation.verifyDbDict(self, 'dbbuilddict', bdict)
self.assertEqual(sorted(bdicts, key=lambda bd: bd['id']),
[self.threeBdicts[50], self.threeBdicts[51]])
def callGet(self, path, resultSpec=None):
self.assertIsInstance(path, tuple)
if resultSpec is None:
resultSpec = resultspec.ResultSpec()
endpoint, kwargs = self.matcher[path]
self.assertIdentical(endpoint, self.ep)
rv = yield endpoint.get(resultSpec, kwargs)
if self.ep.isCollection:
self.assertIsInstance(rv, (list, base.ListResult))
else:
self.assertIsInstance(rv, (dict, type(None)))
return rv
def get(self,
path,
filters=None,
fields=None,
order=None,
limit=None,
offset=None):
resultSpec = resultspec.ResultSpec(filters=filters,
fields=fields,
order=order,
limit=limit,
offset=offset)
return self.get_with_resultspec(path, resultSpec)
def test_get_contains_two_tags_one_unknown(self):
resultSpec = resultspec.ResultSpec(
filters=[resultspec.Filter('tags', 'contains', ["tagA", "tagC"])])
d = self.callGet(('builders', ))
@d.addCallback
def check(builders):
builders = resultSpec.apply(builders)
[self.validateData(b) for b in builders]
self.assertEqual(sorted([b['builderid'] for b in builders]),
[3, 5])
return d
def test_apply_ordering_multi(self):
data = mklist(('fn', 'ln'),
('cedric', 'willis'),
('albert', 'engelbert'),
('bruce', 'willis'),
('dwayne', 'montague'))
exp = base.ListResult(mklist(('fn', 'ln'),
('albert', 'engelbert'),
('dwayne', 'montague'),
('bruce', 'willis'),
('cedric', 'willis')), total=4)
random.shuffle(data)
self.assertListResultEqual(
resultspec.ResultSpec(order=['ln', 'fn']).apply(data),
exp)
exp = base.ListResult(mklist(('fn', 'ln'),
('cedric', 'willis'),
('bruce', 'willis'),
('dwayne', 'montague'),
('albert', 'engelbert')), total=4)
self.assertListResultEqual(
resultspec.ResultSpec(order=['-ln', '-fn']).apply(data),
exp)
def testGetProperties(self):
self.master.db.insertTestData([
fakedb.BuildsetProperty(buildsetid=8822, property_name='prop1',
property_value='["one", "fake1"]'),
fakedb.BuildsetProperty(buildsetid=8822, property_name='prop2',
property_value='["two", "fake2"]'),
])
prop = resultspec.Property(b'property', 'eq', '*')
buildrequests = yield self.callGet(('builders', 78, 'buildrequests'),
resultSpec=resultspec.ResultSpec(properties=[prop]))
self.assertEqual(len(buildrequests), 1)
self.assertEqual(buildrequests[0]['buildrequestid'], 46)
self.assertEqual(buildrequests[0]['properties'],
{u'prop1': (u'one', u'fake1'), u'prop2': (u'two', u'fake2')})
def testGetClaimedByMasterIdFilters(self):
getBuildRequestsMock = mock.Mock(return_value={})
self.patch(self.master.db.buildrequests, 'getBuildRequests',
getBuildRequestsMock)
f1 = resultspec.Filter('claimed', 'eq', [True])
f2 = resultspec.Filter('claimed_by_masterid', 'eq',
[fakedb.FakeBuildRequestsComponent.MASTER_ID])
yield self.callGet(('buildrequests', ),
resultSpec=resultspec.ResultSpec(filters=[f1, f2]))
getBuildRequestsMock.assert_called_with(
builderid=None,
bsid=None,
complete=None,
claimed=fakedb.FakeBuildRequestsComponent.MASTER_ID)
def do_test_pagination(self, bareList):
data = mklist('x', *lrange(101, 131))
if not bareList:
data = base.ListResult(data)
data.offset = None
data.total = len(data)
data.limit = None
self.assertListResultEqual(
resultspec.ResultSpec(offset=0).apply(data),
base.ListResult(mklist('x', *lrange(101, 131)),
offset=0, total=30))
self.assertListResultEqual(
resultspec.ResultSpec(offset=10).apply(data),
base.ListResult(mklist('x', *lrange(111, 131)),
offset=10, total=30))
self.assertListResultEqual(
resultspec.ResultSpec(offset=10, limit=10).apply(data),
base.ListResult(mklist('x', *lrange(111, 121)),
offset=10, total=30, limit=10))
self.assertListResultEqual(
resultspec.ResultSpec(offset=20, limit=15).apply(data),
base.ListResult(mklist('x', *lrange(121, 131)),
offset=20, total=30, limit=15)) # off the end
def testGetFilters(self):
getBuildRequestsMock = mock.Mock(return_value={})
self.patch(self.master.db.buildrequests, 'getBuildRequests',
getBuildRequestsMock)
f1 = resultspec.Filter('complete', 'eq', [False])
f2 = resultspec.Filter('claimed', 'eq', [True])
f3 = resultspec.Filter('buildsetid', 'eq', [55])
f4 = resultspec.Filter('branch', 'eq', ['mybranch'])
f5 = resultspec.Filter('repository', 'eq', ['myrepo'])
yield self.callGet(
('buildrequests', ),
resultSpec=resultspec.ResultSpec(filters=[f1, f2, f3, f4, f5]))
getBuildRequestsMock.assert_called_with(builderid=None,
bsid=55,
complete=False,
claimed=True)
def get(self,
path,
filters=None,
fields=None,
order=None,
limit=None,
offset=None):
resultSpec = resultspec.ResultSpec(filters=filters,
fields=fields,
order=order,
limit=limit,
offset=offset)
endpoint, kwargs = self.getEndpoint(path)
rv = yield endpoint.get(resultSpec, kwargs)
if resultSpec:
rv = resultSpec.apply(rv)
defer.returnValue(rv)
def callGet(self, path, resultSpec=None):
self.assertIsInstance(path, tuple)
if resultSpec is None:
resultSpec = resultspec.ResultSpec()
endpoint, kwargs = self.matcher[path]
self.assertIdentical(endpoint, self.ep)
d = endpoint.get(resultSpec, kwargs)
self.assertIsInstance(d, defer.Deferred)
@d.addCallback
def checkNumber(rv):
if self.ep.isCollection:
self.assertIsInstance(rv, (list, base.ListResult))
else:
self.assertIsInstance(rv, (dict, type(None)))
return rv
return d
def test_getChanges_missing(self):
yield self.insertTestData(self.change13_rows + self.change14_rows)
def check(changes):
# requested all, but only got 2
# sort by changeid, since we assert on change 13 at index 0
changes.sort(key=lambda c: c['changeid'])
changeids = [c['changeid'] for c in changes]
self.assertEqual(changeids, [13, 14])
# double-check that they have .files, etc.
self.assertEqual(sorted(changes[0]['files']),
sorted(['master/README.txt', 'worker/README.txt']))
self.assertEqual(changes[0]['properties'],
{'notest': ('no', 'Change')})
rs = resultspec.ResultSpec(order=['-changeid'], limit=5)
changes = yield self.db.changes.getChanges(resultSpec=rs)
check(changes)
changes = yield self.db.changes.getChanges()
check(changes)
def decodeResultSpec(self, request, endpoint):
reqArgs = request.args
def checkFields(fields, negOk=False):
for k in fields:
if k[0] == '-' and negOk:
k = k[1:]
if k not in entityType.fieldNames:
raise BadRequest("no such field %r" % (k, ))
entityType = endpoint.rtype.entityType
limit = offset = order = fields = None
filters, properties = [], []
for arg in reqArgs:
if arg == 'order':
order = reqArgs[arg]
checkFields(order, True)
continue
elif arg == 'field':
fields = reqArgs[arg]
checkFields(fields, False)
continue
elif arg == 'limit':
try:
limit = int(reqArgs[arg][0])
except Exception:
raise BadRequest('invalid limit')
continue
elif arg == 'offset':
try:
offset = int(reqArgs[arg][0])
except Exception:
raise BadRequest('invalid offset')
continue
elif arg == 'property':
try:
props = [v.decode('utf-8') for v in reqArgs[arg]]
except Exception:
raise BadRequest('invalid property value for %s' % arg)
properties.append(resultspec.Property(arg, 'eq', props))
continue
elif arg in entityType.fieldNames:
field = entityType.fields[arg]
try:
values = [field.valueFromString(v) for v in reqArgs[arg]]
except Exception:
raise BadRequest('invalid filter value for %s' % arg)
filters.append(resultspec.Filter(arg, 'eq', values))
continue
elif '__' in arg:
field, op = arg.rsplit('__', 1)
args = reqArgs[arg]
operators = (resultspec.Filter.singular_operators if len(args)
== 1 else resultspec.Filter.plural_operators)
if op in operators and field in entityType.fieldNames:
fieldType = entityType.fields[field]
try:
values = [
fieldType.valueFromString(v) for v in reqArgs[arg]
]
except Exception:
raise BadRequest('invalid filter value for %s' % arg)
filters.append(resultspec.Filter(field, op, values))
continue
raise BadRequest("unrecognized query parameter '%s'" % (arg, ))
# if ordering or filtering is on a field that's not in fields, bail out
if fields:
fieldsSet = set(fields)
if order and set(order) - fieldsSet:
raise BadRequest("cannot order on un-selected fields")
for filter in filters:
if filter.field not in fieldsSet:
raise BadRequest("cannot filter on un-selected fields")
# build the result spec
rspec = resultspec.ResultSpec(fields=fields,
limit=limit,
offset=offset,
order=order,
filters=filters,
properties=properties)
# for singular endpoints, only allow fields
if not endpoint.isCollection:
if rspec.filters:
raise BadRequest("this is not a collection")
return rspec
def resultspec_from_jsonapi(self, req_args, entityType, is_collection):
def checkFields(fields, negOk=False):
for field in fields:
k = bytes2unicode(field)
if k[0] == '-' and negOk:
k = k[1:]
if k not in entityType.fieldNames:
raise exceptions.InvalidQueryParameter(
"no such field '{}'".format(k))
limit = offset = order = fields = None
filters, properties = [], []
limit = offset = order = fields = None
filters, properties = [], []
for arg in req_args:
argStr = bytes2unicode(arg)
if argStr == 'order':
order = tuple([bytes2unicode(o) for o in req_args[arg]])
checkFields(order, True)
elif argStr == 'field':
fields = req_args[arg]
checkFields(fields, False)
elif argStr == 'limit':
try:
limit = int(req_args[arg][0])
except Exception as e:
raise exceptions.InvalidQueryParameter(
'invalid limit') from e
elif argStr == 'offset':
try:
offset = int(req_args[arg][0])
except Exception as e:
raise exceptions.InvalidQueryParameter(
'invalid offset') from e
elif argStr == 'property':
try:
props = []
for v in req_args[arg]:
if not isinstance(v, (bytes, str)):
raise TypeError("Invalid type {} for {}".format(
type(v), v))
props.append(bytes2unicode(v))
except Exception as e:
raise exceptions.InvalidQueryParameter(
'invalid property value for {}'.format(arg)) from e
properties.append(resultspec.Property(arg, 'eq', props))
elif argStr in entityType.fieldNames:
field = entityType.fields[argStr]
try:
values = [field.valueFromString(v) for v in req_args[arg]]
except Exception as e:
raise exceptions.InvalidQueryParameter(
'invalid filter value for {}'.format(argStr)) from e
filters.append(resultspec.Filter(argStr, 'eq', values))
elif '__' in argStr:
field, op = argStr.rsplit('__', 1)
args = req_args[arg]
operators = (resultspec.Filter.singular_operators if len(args)
== 1 else resultspec.Filter.plural_operators)
if op in operators and field in entityType.fieldNames:
fieldType = entityType.fields[field]
try:
values = [
fieldType.valueFromString(v) for v in req_args[arg]
]
except Exception as e:
raise exceptions.InvalidQueryParameter(
'invalid filter value for {}'.format(
argStr)) from e
filters.append(resultspec.Filter(field, op, values))
else:
raise exceptions.InvalidQueryParameter(
"unrecognized query parameter '{}'".format(argStr))
# if ordering or filtering is on a field that's not in fields, bail out
if fields:
fields = [bytes2unicode(f) for f in fields]
fieldsSet = set(fields)
if order and {o.lstrip('-') for o in order} - fieldsSet:
raise exceptions.InvalidQueryParameter(
"cannot order on un-selected fields")
for filter in filters:
if filter.field not in fieldsSet:
raise exceptions.InvalidQueryParameter(
"cannot filter on un-selected fields")
# build the result spec
rspec = resultspec.ResultSpec(fields=fields,
limit=limit,
offset=offset,
order=order,
filters=filters,
properties=properties)
# for singular endpoints, only allow fields
if not is_collection:
if rspec.filters:
raise exceptions.InvalidQueryParameter(
"this is not a collection")
return rspec
def test_getChangesOtherOrder(self):
resultSpec = resultspec.ResultSpec(limit=1, order=('-when_time_stamp',))
changes = yield self.callGet(('changes',), resultSpec=resultSpec)
self.assertEqual(len(changes), 1)
def test_apply_None(self):
self.assertEqual(resultspec.ResultSpec().apply(None), None)
def test_getChangesOtherOffset(self):
resultSpec = resultspec.ResultSpec(
limit=1, offset=1, order=('-changeid',))
changes = yield self.callGet(('changes',), resultSpec=resultSpec)
self.assertEqual(len(changes), 1)
def test_popField(self):
rs = resultspec.ResultSpec(fields=['foo', 'bar'])
self.assertTrue(rs.popField('foo'))
self.assertEqual(rs.fields, ['bar'])
def test_popField_not_present(self):
rs = resultspec.ResultSpec(fields=['foo', 'bar'])
self.assertFalse(rs.popField('nosuch'))
self.assertEqual(rs.fields, ['foo', 'bar'])
def test_removeOrder(self):
rs = resultspec.ResultSpec(order=['foo', '-bar'])
rs.removeOrder()
self.assertEqual(rs.order, None)
def test_popIntegerFilterNotInt(self):
rs = resultspec.ResultSpec(filters=[
resultspec.Filter('foo', 'eq', ['bar']),
])
with self.assertRaises(ValueError):
rs.popIntegerFilter('foo')
def test_popIntegerFilterSeveral(self):
rs = resultspec.ResultSpec(filters=[
resultspec.Filter('foo', 'eq', ['12', '13']),
])
self.assertEqual(rs.popIntegerFilter('foo'), None)
def test_popStringFilterSeveral(self):
rs = resultspec.ResultSpec(filters=[
resultspec.Filter('foo', 'eq', ['foo', 'bar']),
])
self.assertEqual(rs.popStringFilter('foo'), None)
def test_getChangesOtherOrder(self):
resultSpec = resultspec.ResultSpec(limit=1, order=['-when_time_stamp'])
changes = yield self.callGet(('changes',), resultSpec=resultSpec)
# limit not implemented for other order
self.assertEqual(len(changes), 2)