def test_02_test_count(self): # compute the record counts for each job + comp counts = {} comps = {} for job_id in range(1, 6): filt = Sos.Filter(self.schema['job_time_cond']) filt.add_condition(self.schema['job_id'], Sos.COND_EQ, job_id) count = 0 o = filt.begin() comp_id = o['component_id'] comps[job_id] = comp_id filt.add_condition(self.schema['component_id'], Sos.COND_EQ, comp_id) o = filt.begin() while o: count += 1 o = filt.next() counts[job_id] = count del filt # confirm that the count returned by filt.count() matches # the computed count for job_id in range(1, 6): filt = Sos.Filter(self.schema['job_time_cond']) filt.add_condition(self.schema['job_id'], Sos.COND_EQ, job_id) filt.add_condition(self.schema['component_id'], Sos.COND_EQ, comps[job_id]) count = filt.count() self.assertEqual(count, counts[job_id])
def __test_prev(self, attr_name): global key global data attr = self.schema[attr_name] f = Sos.Filter(attr) # Iterate to the start count = len(data) o = f.end() while o: d = data[count-1] v = o[:] self.assertEqual(d[0], v[0]) self.assertEqual(d[1], v[1]) count -= 1 o = f.prev() self.assertEqual( count, 0 ) # Add more data key = 500 # Put the key before the 1st key of the last test new_data = [] key, new_data = self.__add_data( key, new_data ) count = len(new_data) # f.prev should return the new data o = f.prev() self.assertIsNotNone( o ) while o: d = new_data[count-1] v = o[:] self.assertEqual(d[0], v[0]) self.assertEqual(d[1], v[1]) count -= 1 o = f.prev() # we should not see any object twice self.assertEqual( count, 0 )
def setUp(self): self.min_a_1 = 1000 self.max_a_1 = 10000 self.filt = Sos.Filter(self.schema.attr_by_name('join_key')) a_1 = self.schema.attr_by_name('a_1') self.filt.add_condition(a_1, Sos.COND_GE, self.min_a_1) self.filt.add_condition(a_1, Sos.COND_LE, self.max_a_1)
def test_uint64_k0_k1_k2_k3(self): a_join = self.uint64_schema.attr_by_name('a_join') k0 = self.uint64_schema.attr_by_name('k0') k1 = self.uint64_schema.attr_by_name('k1') k2 = self.uint64_schema.attr_by_name('k2') k3 = self.uint64_schema.attr_by_name('k3') f = Sos.Filter(a_join) f.add_condition(k0, Sos.COND_GE, 4) f.add_condition(k0, Sos.COND_LE, 12) f.add_condition(k1, Sos.COND_GE, 4) f.add_condition(k1, Sos.COND_LE, 12) f.add_condition(k2, Sos.COND_GE, 4) f.add_condition(k2, Sos.COND_LE, 12) f.add_condition(k3, Sos.COND_GE, 4) f.add_condition(k3, Sos.COND_LE, 12) o = f.begin() count = 0 while o: # Dprint(o[:]) count += 1 o = next(f) Dprint("Misses {0}".format(f.miss_count())) Dprint("count {0}".format(count)) self.assertTrue(f.miss_count() <= 4) self.assertEqual(count, 9 * 9 * 9 * 9) del f
def test_int32_k0_k1_k2_k3_prev(self): a_join = self.int32_schema.attr_by_name('a_join') k0 = self.int32_schema.attr_by_name('k0') k1 = self.int32_schema.attr_by_name('k1') k2 = self.int32_schema.attr_by_name('k2') k3 = self.int32_schema.attr_by_name('k3') f = Sos.Filter(a_join) f.add_condition(k0, Sos.COND_LE, -4) f.add_condition(k0, Sos.COND_GE, -12) f.add_condition(k1, Sos.COND_LE, -4) f.add_condition(k1, Sos.COND_GE, -12) f.add_condition(k2, Sos.COND_LE, -4) f.add_condition(k2, Sos.COND_GE, -12) f.add_condition(k3, Sos.COND_LE, -4) f.add_condition(k3, Sos.COND_GE, -12) o = f.end() count = 0 while o: # Dprint(o[:]) count += 1 o = f.prev() Dprint("Misses {0}".format(f.miss_count())) Dprint("Count {0}".format(count)) self.assertTrue(f.miss_count() <= 4) self.assertEqual(count, 9 * 9 * 9 * 9) del f
def __join_test_next_prev(self, join_attr_name, attr_name, min_v, max_v): join_attr = self.schema[join_attr_name] attr = self.schema[attr_name] f = Sos.Filter(join_attr) f.add_condition(attr, Sos.COND_GE, min_v) f.add_condition(attr, Sos.COND_LE, max_v) o = f.begin() next_count = 0 while o: Dprint(o[:]) next_count += 1 self.assertTrue(o[attr_name] >= min_v) self.assertTrue(o[attr_name] <= max_v) o = next(f) # iterate backwards, the count should be the same o = f.end() prev_count = 0 while o: Dprint(o[:]) prev_count += 1 self.assertTrue(o[attr_name] >= min_v) self.assertTrue(o[attr_name] <= max_v) o = f.prev() self.assertEqual(next_count, prev_count)
def __test_next_prev(self, attr_name, min_v, max_v): attr = self.schema[attr_name] f = Sos.Filter(attr) f.add_condition(attr, Sos.COND_GE, min_v) f.add_condition(attr, Sos.COND_LE, max_v) o = f.begin() next_count = 0 while o: next_count += 1 v = o[attr_name] if type(v) == numpy.ndarray: v = v.tolist() Dprint("{0} >= {1}".format(v, min_v)) Dprint("{0} <= {1}".format(v, max_v)) self.assertTrue(v >= min_v) self.assertTrue(v <= max_v) o = next(f) self.assertTrue(next_count > 0) # iterate backwards, the count should be the same o = f.end() prev_count = 0 while o: prev_count += 1 v = o[attr_name] if type(v) == numpy.ndarray: v = v.tolist() Dprint("{0} >= {1}".format(v, min_v)) Dprint("{0} <= {1}".format(v, max_v)) self.assertTrue(v >= min_v) self.assertTrue(v <= max_v) o = f.prev() self.assertTrue(prev_count > 0) self.assertEqual(next_count, prev_count)
def test_10_filter_fwd(self): ts_attr = self.schema['timestamp'] for i in range(len(data_float)): f = Sos.Filter(ts_attr) f.add_condition(ts_attr, Sos.COND_GE, data_float[i][1]) Dprint(data_float[i][1]) o = f.begin() count = len(data_float) - i while o: count -= 1 Dprint(o[:]) o = next(f) self.assertEqual( count, 0 ) Dprint("--------------------------------")
def test_11_filter_rev(self): ts_attr = self.schema['timestamp'] for i in range(len(data_float) -1, -1, -1): f = Sos.Filter(ts_attr) f.add_condition(ts_attr, Sos.COND_GE, data_float[0][1]) f.add_condition(ts_attr, Sos.COND_LE, data_float[i][1]) Dprint(data_float[0][1], data_float[i][1]) o = f.end() count = i + 1 while o: count -= 1 Dprint(o[:]) o = f.prev() self.assertEqual( count, 0 ) Dprint("--------------------------------")
def test_uint16_k0(self): a_join = self.uint16_schema.attr_by_name('a_join') k0 = self.uint16_schema.attr_by_name('k0') f = Sos.Filter(a_join) f.add_condition(k0, Sos.COND_GE, 4) f.add_condition(k0, Sos.COND_LE, 12) o = f.begin() count = 0 while o: # Dprint(o[:]) count += 1 o = next(f) Dprint("Misses {0}".format(f.miss_count())) Dprint("Count {0}".format(count)) self.assertTrue(f.miss_count() <= 1) self.assertEqual(count, 9 * 16 * 16 * 16) del f
def test_int64_k0_rev(self): a_join = self.int64_schema.attr_by_name('a_join') k0 = self.int64_schema.attr_by_name('k0') f = Sos.Filter(a_join) f.add_condition(k0, Sos.COND_LE, -4) f.add_condition(k0, Sos.COND_GE, -12) o = f.end() count = 0 while o: # Dprint(o[:]) count += 1 o = f.prev() Dprint("Misses {0}".format(f.miss_count())) Dprint("Count {0}".format(count)) self.assertTrue(f.miss_count() <= 1) self.assertEqual(count, 9 * 17 * 17 * 17) del f
def test_int32_k0_k1(self): a_join = self.int32_schema.attr_by_name('a_join') k0 = self.int32_schema.attr_by_name('k0') k1 = self.int32_schema.attr_by_name('k1') f = Sos.Filter(a_join) f.add_condition(k0, Sos.COND_LE, -4) f.add_condition(k0, Sos.COND_GE, -12) f.add_condition(k1, Sos.COND_LE, -4) f.add_condition(k1, Sos.COND_GE, -12) o = f.begin() count = 0 while o: # Dprint(o[:]) count += 1 o = next(f) Dprint("Misses {0}".format(f.miss_count())) Dprint("count {0}".format(count)) self.assertTrue(f.miss_count() <= 2) self.assertEqual(count, 9 * 9 * 17 * 17) del f
def select(self, comp_id=None, job_id=None): self.timestamp = self.in_schema.attr_by_name("timestamp") if self.index is None: if job_id is not None: self.index = "job_comp_time" elif comp_id is not None: self.index = "comp_time" else: self.index = "timestamp" self.filter = Sos.Filter(self.in_schema.attr_by_name(self.index)) self.filter.add_condition(self.timestamp, Sos.COND_GE, self.start.strftime(self.dt_fmt)) self.filter.add_condition(self.timestamp, Sos.COND_LE, self.end.strftime(self.dt_fmt)) if job_id is not None: self.job_id = self.in_schema.attr_by_name("job_id") self.filter.add_condition(self.job_id, Sos.COND_EQ, str(job_id)) if comp_id is not None: self.comp_id = self.in_schema.attr_by_name("component_id") self.filter.add_condition(self.comp_id, Sos.COND_EQ, str(comp_id))
def __query_test(self, index, job_id, comp_id, start, end, expected_count): join = self.schema.attr_by_name(index) time_a = self.schema['timestamp'] f = Sos.Filter(join) if job_id is not None: job_a = self.schema['job_id'] f.add_condition(job_a, Sos.COND_EQ, job_id) if comp_id is not None: comp_a = self.schema['component_id'] f.add_condition(comp_a, Sos.COND_EQ, comp_id) f.add_condition(time_a, Sos.COND_GE, start) f.add_condition(time_a, Sos.COND_LT, end) o = f.begin() count = 0 while o: # Dprint(o[:4]) count += 1 o = next(f) Dprint("Misses {0}".format(f.miss_count())) Dprint("Count {0}".format(count)) self.assertTrue(f.miss_count() <= 3) self.assertEqual(count, expected_count) del f
def __test_next(self, attr_name): global key global data key, data = self.__add_data(key, data) attr = self.schema[attr_name] f = Sos.Filter(attr) # Iterate to the end count = 0 o = f.begin() while o: d = data[count] v = o[:] self.assertEqual(d[0], v[0]) self.assertEqual(d[1], v[1]) count += 1 o = next(f) self.assertEqual(count, len(data)) # Add more data new_data = [] key, new_data = self.__add_data( key, new_data ) # next(f) should return the new data o = next(f) self.assertIsNotNone( o ) count = 0 while o: d = new_data[count] v = o[:] self.assertEqual(d[0], v[0]) self.assertEqual(d[1], v[1]) count += 1 o = next(f) # we should not see any object twice self.assertEqual( count, len(new_data) ) # Make the global data match the container data = data + new_data
def parse_query(self, request): self.parms = request.GET self.request = request try: self.parse_request(self.parms) except Exception as e: return (1, "Exception in parse_request: {0}".format(e), None) if not self.schema(): return (1, "A 'schema' clause must be specified.\n", None) if not 'index' in self.parms: return (1, "An 'index' clause must be specified.\n", None) # # Open an iterator on the container # self.index_attr = None self.index_name = self.parms['index'] if 'indexName' not in self.request.session: self.reset() if 'containerName' not in self.request.session: self.reset() self.schema_name = self.schema().name() self.index_attr = self.schema().attr_by_name(self.index_name) self.iter_ = self.index_attr.index().stats() self.filt = Sos.Filter(self.index_attr) if 'unique' in self.parms: self.unique = True self.filt.unique() else: self.unique = False self.card = self.iter_['cardinality'] if self.unique: self.card = self.card - self.iter_['duplicates'] # # Parse the select clause. The view_cols contains the index as it's first element. # self.view_cols = [] if 'select' in self.parms: self.select = self.parms['select'] for attr_name in self.select.split(','): if attr_name != self.index_name: self.view_cols.append(attr_name) else: for attr in self.schema(): if attr.name() != self.index_name: self.view_cols.append(attr.name()) # # Parse the where clause # # # A filter is an array of conditions # if 'where' in self.parms: where = self.parms['where'] conds = where.split(',') for cond in conds: tokens = cond.split(':') if len(tokens) < 3: return (1, "Invalid where clause '{0}', " "valid syntax is attr:cmp_str:value".format(cond), None) attr_name = tokens[0] attr = self.schema().attr_by_name(attr_name) if not attr: return (1, "The attribute {0} was not found " "in the schema {1}".format(attr_name, self.schema().name()), None) sos_cmp = sos_filt_cond[tokens[1]] value_str = None tokens[2] = tokens[2].split('"')[1] for s in tokens[2:]: if value_str: value_str = value_str + ':' + s else: value_str = s self.filt.add_condition(attr, sos_cmp, int(tokens[2])) obj = None if self.start == 0: self.reset() obj = self.filt.begin() skip = self.start elif self.start + self.count >= self.card: self.reset() obj = self.filt.end() skip = self.card % self.count while obj and skip > 0: obj = self.filt.prev() skip = skip - 1 else: self.filt.set_pos(str(self.request.session['pos'])) skip = self.start - self.request.session['recordNo'] obj = self.filt.obj() while obj and skip != 0: if skip > 0: obj = next(self.filt) skip = skip - 1 else: obj = self.filt.prev() skip = skip + 1 pos = self.filt.get_pos() self.request.session['pos'] = pos self.request.session['recordNo'] = self.start return (0, None, obj)
def setUp(self): self.filt = Sos.Filter(self.schema['job_time_comp'])