def test_02_test_count(self):
# compute the record counts for each job + comp
counts = {}
comps = {}
for job_id in range(1, 6):
filt = Sos.Filter(self.schema['job_time_cond'])
filt.add_condition(self.schema['job_id'], Sos.COND_EQ, job_id)
count = 0
o = filt.begin()
comp_id = o['component_id']
comps[job_id] = comp_id
filt.add_condition(self.schema['component_id'], Sos.COND_EQ,
comp_id)
o = filt.begin()
while o:
count += 1
o = filt.next()
counts[job_id] = count
del filt
# confirm that the count returned by filt.count() matches
# the computed count
for job_id in range(1, 6):
filt = Sos.Filter(self.schema['job_time_cond'])
filt.add_condition(self.schema['job_id'], Sos.COND_EQ, job_id)
filt.add_condition(self.schema['component_id'], Sos.COND_EQ,
comps[job_id])
count = filt.count()
self.assertEqual(count, counts[job_id])
def __test_prev(self, attr_name):
global key
global data
attr = self.schema[attr_name]
f = Sos.Filter(attr)
# Iterate to the start
count = len(data)
o = f.end()
while o:
d = data[count-1]
v = o[:]
self.assertEqual(d[0], v[0])
self.assertEqual(d[1], v[1])
count -= 1
o = f.prev()
self.assertEqual( count, 0 )
# Add more data
key = 500 # Put the key before the 1st key of the last test
new_data = []
key, new_data = self.__add_data( key, new_data )
count = len(new_data)
# f.prev should return the new data
o = f.prev()
self.assertIsNotNone( o )
while o:
d = new_data[count-1]
v = o[:]
self.assertEqual(d[0], v[0])
self.assertEqual(d[1], v[1])
count -= 1
o = f.prev()
# we should not see any object twice
self.assertEqual( count, 0 )
def setUp(self):
self.min_a_1 = 1000
self.max_a_1 = 10000
self.filt = Sos.Filter(self.schema.attr_by_name('join_key'))
a_1 = self.schema.attr_by_name('a_1')
self.filt.add_condition(a_1, Sos.COND_GE, self.min_a_1)
self.filt.add_condition(a_1, Sos.COND_LE, self.max_a_1)
def test_uint64_k0_k1_k2_k3(self):
a_join = self.uint64_schema.attr_by_name('a_join')
k0 = self.uint64_schema.attr_by_name('k0')
k1 = self.uint64_schema.attr_by_name('k1')
k2 = self.uint64_schema.attr_by_name('k2')
k3 = self.uint64_schema.attr_by_name('k3')
f = Sos.Filter(a_join)
f.add_condition(k0, Sos.COND_GE, 4)
f.add_condition(k0, Sos.COND_LE, 12)
f.add_condition(k1, Sos.COND_GE, 4)
f.add_condition(k1, Sos.COND_LE, 12)
f.add_condition(k2, Sos.COND_GE, 4)
f.add_condition(k2, Sos.COND_LE, 12)
f.add_condition(k3, Sos.COND_GE, 4)
f.add_condition(k3, Sos.COND_LE, 12)
o = f.begin()
count = 0
while o:
# Dprint(o[:])
count += 1
o = next(f)
Dprint("Misses {0}".format(f.miss_count()))
Dprint("count {0}".format(count))
self.assertTrue(f.miss_count() <= 4)
self.assertEqual(count, 9 * 9 * 9 * 9)
del f
def test_int32_k0_k1_k2_k3_prev(self):
a_join = self.int32_schema.attr_by_name('a_join')
k0 = self.int32_schema.attr_by_name('k0')
k1 = self.int32_schema.attr_by_name('k1')
k2 = self.int32_schema.attr_by_name('k2')
k3 = self.int32_schema.attr_by_name('k3')
f = Sos.Filter(a_join)
f.add_condition(k0, Sos.COND_LE, -4)
f.add_condition(k0, Sos.COND_GE, -12)
f.add_condition(k1, Sos.COND_LE, -4)
f.add_condition(k1, Sos.COND_GE, -12)
f.add_condition(k2, Sos.COND_LE, -4)
f.add_condition(k2, Sos.COND_GE, -12)
f.add_condition(k3, Sos.COND_LE, -4)
f.add_condition(k3, Sos.COND_GE, -12)
o = f.end()
count = 0
while o:
# Dprint(o[:])
count += 1
o = f.prev()
Dprint("Misses {0}".format(f.miss_count()))
Dprint("Count {0}".format(count))
self.assertTrue(f.miss_count() <= 4)
self.assertEqual(count, 9 * 9 * 9 * 9)
del f
def __join_test_next_prev(self, join_attr_name, attr_name, min_v, max_v):
join_attr = self.schema[join_attr_name]
attr = self.schema[attr_name]
f = Sos.Filter(join_attr)
f.add_condition(attr, Sos.COND_GE, min_v)
f.add_condition(attr, Sos.COND_LE, max_v)
o = f.begin()
next_count = 0
while o:
Dprint(o[:])
next_count += 1
self.assertTrue(o[attr_name] >= min_v)
self.assertTrue(o[attr_name] <= max_v)
o = next(f)
# iterate backwards, the count should be the same
o = f.end()
prev_count = 0
while o:
Dprint(o[:])
prev_count += 1
self.assertTrue(o[attr_name] >= min_v)
self.assertTrue(o[attr_name] <= max_v)
o = f.prev()
self.assertEqual(next_count, prev_count)
def __test_next_prev(self, attr_name, min_v, max_v):
attr = self.schema[attr_name]
f = Sos.Filter(attr)
f.add_condition(attr, Sos.COND_GE, min_v)
f.add_condition(attr, Sos.COND_LE, max_v)
o = f.begin()
next_count = 0
while o:
next_count += 1
v = o[attr_name]
if type(v) == numpy.ndarray:
v = v.tolist()
Dprint("{0} >= {1}".format(v, min_v))
Dprint("{0} <= {1}".format(v, max_v))
self.assertTrue(v >= min_v)
self.assertTrue(v <= max_v)
o = next(f)
self.assertTrue(next_count > 0)
# iterate backwards, the count should be the same
o = f.end()
prev_count = 0
while o:
prev_count += 1
v = o[attr_name]
if type(v) == numpy.ndarray:
v = v.tolist()
Dprint("{0} >= {1}".format(v, min_v))
Dprint("{0} <= {1}".format(v, max_v))
self.assertTrue(v >= min_v)
self.assertTrue(v <= max_v)
o = f.prev()
self.assertTrue(prev_count > 0)
self.assertEqual(next_count, prev_count)
def test_10_filter_fwd(self):
ts_attr = self.schema['timestamp']
for i in range(len(data_float)):
f = Sos.Filter(ts_attr)
f.add_condition(ts_attr, Sos.COND_GE, data_float[i][1])
Dprint(data_float[i][1])
o = f.begin()
count = len(data_float) - i
while o:
count -= 1
Dprint(o[:])
o = next(f)
self.assertEqual( count, 0 )
Dprint("--------------------------------")
def test_11_filter_rev(self):
ts_attr = self.schema['timestamp']
for i in range(len(data_float) -1, -1, -1):
f = Sos.Filter(ts_attr)
f.add_condition(ts_attr, Sos.COND_GE, data_float[0][1])
f.add_condition(ts_attr, Sos.COND_LE, data_float[i][1])
Dprint(data_float[0][1], data_float[i][1])
o = f.end()
count = i + 1
while o:
count -= 1
Dprint(o[:])
o = f.prev()
self.assertEqual( count, 0 )
Dprint("--------------------------------")
def test_uint16_k0(self):
a_join = self.uint16_schema.attr_by_name('a_join')
k0 = self.uint16_schema.attr_by_name('k0')
f = Sos.Filter(a_join)
f.add_condition(k0, Sos.COND_GE, 4)
f.add_condition(k0, Sos.COND_LE, 12)
o = f.begin()
count = 0
while o:
# Dprint(o[:])
count += 1
o = next(f)
Dprint("Misses {0}".format(f.miss_count()))
Dprint("Count {0}".format(count))
self.assertTrue(f.miss_count() <= 1)
self.assertEqual(count, 9 * 16 * 16 * 16)
del f
def test_int64_k0_rev(self):
a_join = self.int64_schema.attr_by_name('a_join')
k0 = self.int64_schema.attr_by_name('k0')
f = Sos.Filter(a_join)
f.add_condition(k0, Sos.COND_LE, -4)
f.add_condition(k0, Sos.COND_GE, -12)
o = f.end()
count = 0
while o:
# Dprint(o[:])
count += 1
o = f.prev()
Dprint("Misses {0}".format(f.miss_count()))
Dprint("Count {0}".format(count))
self.assertTrue(f.miss_count() <= 1)
self.assertEqual(count, 9 * 17 * 17 * 17)
del f
def test_int32_k0_k1(self):
a_join = self.int32_schema.attr_by_name('a_join')
k0 = self.int32_schema.attr_by_name('k0')
k1 = self.int32_schema.attr_by_name('k1')
f = Sos.Filter(a_join)
f.add_condition(k0, Sos.COND_LE, -4)
f.add_condition(k0, Sos.COND_GE, -12)
f.add_condition(k1, Sos.COND_LE, -4)
f.add_condition(k1, Sos.COND_GE, -12)
o = f.begin()
count = 0
while o:
# Dprint(o[:])
count += 1
o = next(f)
Dprint("Misses {0}".format(f.miss_count()))
Dprint("count {0}".format(count))
self.assertTrue(f.miss_count() <= 2)
self.assertEqual(count, 9 * 9 * 17 * 17)
del f
def select(self, comp_id=None, job_id=None):
self.timestamp = self.in_schema.attr_by_name("timestamp")
if self.index is None:
if job_id is not None:
self.index = "job_comp_time"
elif comp_id is not None:
self.index = "comp_time"
else:
self.index = "timestamp"
self.filter = Sos.Filter(self.in_schema.attr_by_name(self.index))
self.filter.add_condition(self.timestamp,
Sos.COND_GE,
self.start.strftime(self.dt_fmt))
self.filter.add_condition(self.timestamp,
Sos.COND_LE,
self.end.strftime(self.dt_fmt))
if job_id is not None:
self.job_id = self.in_schema.attr_by_name("job_id")
self.filter.add_condition(self.job_id, Sos.COND_EQ, str(job_id))
if comp_id is not None:
self.comp_id = self.in_schema.attr_by_name("component_id")
self.filter.add_condition(self.comp_id, Sos.COND_EQ, str(comp_id))
def __query_test(self, index, job_id, comp_id, start, end, expected_count):
join = self.schema.attr_by_name(index)
time_a = self.schema['timestamp']
f = Sos.Filter(join)
if job_id is not None:
job_a = self.schema['job_id']
f.add_condition(job_a, Sos.COND_EQ, job_id)
if comp_id is not None:
comp_a = self.schema['component_id']
f.add_condition(comp_a, Sos.COND_EQ, comp_id)
f.add_condition(time_a, Sos.COND_GE, start)
f.add_condition(time_a, Sos.COND_LT, end)
o = f.begin()
count = 0
while o:
# Dprint(o[:4])
count += 1
o = next(f)
Dprint("Misses {0}".format(f.miss_count()))
Dprint("Count {0}".format(count))
self.assertTrue(f.miss_count() <= 3)
self.assertEqual(count, expected_count)
del f
def __test_next(self, attr_name):
global key
global data
key, data = self.__add_data(key, data)
attr = self.schema[attr_name]
f = Sos.Filter(attr)
# Iterate to the end
count = 0
o = f.begin()
while o:
d = data[count]
v = o[:]
self.assertEqual(d[0], v[0])
self.assertEqual(d[1], v[1])
count += 1
o = next(f)
self.assertEqual(count, len(data))
# Add more data
new_data = []
key, new_data = self.__add_data( key, new_data )
# next(f) should return the new data
o = next(f)
self.assertIsNotNone( o )
count = 0
while o:
d = new_data[count]
v = o[:]
self.assertEqual(d[0], v[0])
self.assertEqual(d[1], v[1])
count += 1
o = next(f)
# we should not see any object twice
self.assertEqual( count, len(new_data) )
# Make the global data match the container
data = data + new_data
def parse_query(self, request):
self.parms = request.GET
self.request = request
try:
self.parse_request(self.parms)
except Exception as e:
return (1, "Exception in parse_request: {0}".format(e), None)
if not self.schema():
return (1, "A 'schema' clause must be specified.\n", None)
if not 'index' in self.parms:
return (1, "An 'index' clause must be specified.\n", None)
#
# Open an iterator on the container
#
self.index_attr = None
self.index_name = self.parms['index']
if 'indexName' not in self.request.session:
self.reset()
if 'containerName' not in self.request.session:
self.reset()
self.schema_name = self.schema().name()
self.index_attr = self.schema().attr_by_name(self.index_name)
self.iter_ = self.index_attr.index().stats()
self.filt = Sos.Filter(self.index_attr)
if 'unique' in self.parms:
self.unique = True
self.filt.unique()
else:
self.unique = False
self.card = self.iter_['cardinality']
if self.unique:
self.card = self.card - self.iter_['duplicates']
#
# Parse the select clause. The view_cols contains the index as it's first element.
#
self.view_cols = []
if 'select' in self.parms:
self.select = self.parms['select']
for attr_name in self.select.split(','):
if attr_name != self.index_name:
self.view_cols.append(attr_name)
else:
for attr in self.schema():
if attr.name() != self.index_name:
self.view_cols.append(attr.name())
#
# Parse the where clause
#
#
# A filter is an array of conditions
#
if 'where' in self.parms:
where = self.parms['where']
conds = where.split(',')
for cond in conds:
tokens = cond.split(':')
if len(tokens) < 3:
return (1, "Invalid where clause '{0}', "
"valid syntax is attr:cmp_str:value".format(cond),
None)
attr_name = tokens[0]
attr = self.schema().attr_by_name(attr_name)
if not attr:
return (1, "The attribute {0} was not found "
"in the schema {1}".format(attr_name,
self.schema().name()),
None)
sos_cmp = sos_filt_cond[tokens[1]]
value_str = None
tokens[2] = tokens[2].split('"')[1]
for s in tokens[2:]:
if value_str:
value_str = value_str + ':' + s
else:
value_str = s
self.filt.add_condition(attr, sos_cmp, int(tokens[2]))
obj = None
if self.start == 0:
self.reset()
obj = self.filt.begin()
skip = self.start
elif self.start + self.count >= self.card:
self.reset()
obj = self.filt.end()
skip = self.card % self.count
while obj and skip > 0:
obj = self.filt.prev()
skip = skip - 1
else:
self.filt.set_pos(str(self.request.session['pos']))
skip = self.start - self.request.session['recordNo']
obj = self.filt.obj()
while obj and skip != 0:
if skip > 0:
obj = next(self.filt)
skip = skip - 1
else:
obj = self.filt.prev()
skip = skip + 1
pos = self.filt.get_pos()
self.request.session['pos'] = pos
self.request.session['recordNo'] = self.start
return (0, None, obj)
def setUp(self):
self.filt = Sos.Filter(self.schema['job_time_comp'])