コード例 #1
0
    def test_02_test_count(self):
        # compute the record counts for each job + comp
        counts = {}
        comps = {}
        for job_id in range(1, 6):
            filt = Sos.Filter(self.schema['job_time_cond'])
            filt.add_condition(self.schema['job_id'], Sos.COND_EQ, job_id)
            count = 0
            o = filt.begin()
            comp_id = o['component_id']
            comps[job_id] = comp_id
            filt.add_condition(self.schema['component_id'], Sos.COND_EQ,
                               comp_id)
            o = filt.begin()
            while o:
                count += 1
                o = filt.next()
            counts[job_id] = count
            del filt

        # confirm that the count returned by filt.count() matches
        # the computed count
        for job_id in range(1, 6):
            filt = Sos.Filter(self.schema['job_time_cond'])
            filt.add_condition(self.schema['job_id'], Sos.COND_EQ, job_id)
            filt.add_condition(self.schema['component_id'], Sos.COND_EQ,
                               comps[job_id])
            count = filt.count()
            self.assertEqual(count, counts[job_id])
コード例 #2
0
    def __test_prev(self, attr_name):
        global key
        global data
        attr = self.schema[attr_name]
        f = Sos.Filter(attr)
        # Iterate to the start
        count = len(data)
        o = f.end()
        while o:
            d = data[count-1]
            v = o[:]
            self.assertEqual(d[0], v[0])
            self.assertEqual(d[1], v[1])
            count -= 1
            o = f.prev()
        self.assertEqual( count, 0 )

        # Add more data
        key = 500   # Put the key before the 1st key of the last test
        new_data = []
        key, new_data = self.__add_data( key, new_data )
        count = len(new_data)

        # f.prev should return the new data
        o = f.prev()
        self.assertIsNotNone( o )
        while o:
            d = new_data[count-1]
            v = o[:]
            self.assertEqual(d[0], v[0])
            self.assertEqual(d[1], v[1])
            count -= 1
            o = f.prev()
        # we should not see any object twice
        self.assertEqual( count, 0 )
コード例 #3
0
ファイル: filter_test.py プロジェクト: opengridcomputing/SOS
 def setUp(self):
     self.min_a_1 = 1000
     self.max_a_1 = 10000
     self.filt = Sos.Filter(self.schema.attr_by_name('join_key'))
     a_1 = self.schema.attr_by_name('a_1')
     self.filt.add_condition(a_1, Sos.COND_GE, self.min_a_1)
     self.filt.add_condition(a_1, Sos.COND_LE, self.max_a_1)
コード例 #4
0
 def test_uint64_k0_k1_k2_k3(self):
     a_join = self.uint64_schema.attr_by_name('a_join')
     k0 = self.uint64_schema.attr_by_name('k0')
     k1 = self.uint64_schema.attr_by_name('k1')
     k2 = self.uint64_schema.attr_by_name('k2')
     k3 = self.uint64_schema.attr_by_name('k3')
     f = Sos.Filter(a_join)
     f.add_condition(k0, Sos.COND_GE, 4)
     f.add_condition(k0, Sos.COND_LE, 12)
     f.add_condition(k1, Sos.COND_GE, 4)
     f.add_condition(k1, Sos.COND_LE, 12)
     f.add_condition(k2, Sos.COND_GE, 4)
     f.add_condition(k2, Sos.COND_LE, 12)
     f.add_condition(k3, Sos.COND_GE, 4)
     f.add_condition(k3, Sos.COND_LE, 12)
     o = f.begin()
     count = 0
     while o:
         # Dprint(o[:])
         count += 1
         o = next(f)
     Dprint("Misses {0}".format(f.miss_count()))
     Dprint("count {0}".format(count))
     self.assertTrue(f.miss_count() <= 4)
     self.assertEqual(count, 9 * 9 * 9 * 9)
     del f
コード例 #5
0
 def test_int32_k0_k1_k2_k3_prev(self):
     a_join = self.int32_schema.attr_by_name('a_join')
     k0 = self.int32_schema.attr_by_name('k0')
     k1 = self.int32_schema.attr_by_name('k1')
     k2 = self.int32_schema.attr_by_name('k2')
     k3 = self.int32_schema.attr_by_name('k3')
     f = Sos.Filter(a_join)
     f.add_condition(k0, Sos.COND_LE, -4)
     f.add_condition(k0, Sos.COND_GE, -12)
     f.add_condition(k1, Sos.COND_LE, -4)
     f.add_condition(k1, Sos.COND_GE, -12)
     f.add_condition(k2, Sos.COND_LE, -4)
     f.add_condition(k2, Sos.COND_GE, -12)
     f.add_condition(k3, Sos.COND_LE, -4)
     f.add_condition(k3, Sos.COND_GE, -12)
     o = f.end()
     count = 0
     while o:
         # Dprint(o[:])
         count += 1
         o = f.prev()
     Dprint("Misses {0}".format(f.miss_count()))
     Dprint("Count {0}".format(count))
     self.assertTrue(f.miss_count() <= 4)
     self.assertEqual(count, 9 * 9 * 9 * 9)
     del f
コード例 #6
0
    def __join_test_next_prev(self, join_attr_name, attr_name, min_v, max_v):
        join_attr = self.schema[join_attr_name]
        attr = self.schema[attr_name]
        f = Sos.Filter(join_attr)
        f.add_condition(attr, Sos.COND_GE, min_v)
        f.add_condition(attr, Sos.COND_LE, max_v)
        o = f.begin()
        next_count = 0
        while o:
            Dprint(o[:])
            next_count += 1
            self.assertTrue(o[attr_name] >= min_v)
            self.assertTrue(o[attr_name] <= max_v)
            o = next(f)

        # iterate backwards, the count should be the same
        o = f.end()
        prev_count = 0
        while o:
            Dprint(o[:])
            prev_count += 1
            self.assertTrue(o[attr_name] >= min_v)
            self.assertTrue(o[attr_name] <= max_v)
            o = f.prev()

        self.assertEqual(next_count, prev_count)
コード例 #7
0
    def __test_next_prev(self, attr_name, min_v, max_v):
        attr = self.schema[attr_name]
        f = Sos.Filter(attr)
        f.add_condition(attr, Sos.COND_GE, min_v)
        f.add_condition(attr, Sos.COND_LE, max_v)
        o = f.begin()
        next_count = 0
        while o:
            next_count += 1
            v = o[attr_name]
            if type(v) == numpy.ndarray:
                v = v.tolist()
            Dprint("{0} >= {1}".format(v, min_v))
            Dprint("{0} <= {1}".format(v, max_v))
            self.assertTrue(v >= min_v)
            self.assertTrue(v <= max_v)
            o = next(f)
        self.assertTrue(next_count > 0)

        # iterate backwards, the count should be the same
        o = f.end()
        prev_count = 0
        while o:
            prev_count += 1
            v = o[attr_name]
            if type(v) == numpy.ndarray:
                v = v.tolist()
            Dprint("{0} >= {1}".format(v, min_v))
            Dprint("{0} <= {1}".format(v, max_v))
            self.assertTrue(v >= min_v)
            self.assertTrue(v <= max_v)
            o = f.prev()
        self.assertTrue(prev_count > 0)

        self.assertEqual(next_count, prev_count)
コード例 #8
0
ファイル: timestamp_test.py プロジェクト: ovis-hpc/sos
 def test_10_filter_fwd(self):
     ts_attr = self.schema['timestamp']
     for i in range(len(data_float)):
         f = Sos.Filter(ts_attr)
         f.add_condition(ts_attr, Sos.COND_GE, data_float[i][1])
         Dprint(data_float[i][1])
         o = f.begin()
         count = len(data_float) - i
         while o:
             count -= 1
             Dprint(o[:])
             o = next(f)
         self.assertEqual( count, 0 )
         Dprint("--------------------------------")
コード例 #9
0
ファイル: timestamp_test.py プロジェクト: ovis-hpc/sos
 def test_11_filter_rev(self):
     ts_attr = self.schema['timestamp']
     for i in range(len(data_float) -1, -1, -1):
         f = Sos.Filter(ts_attr)
         f.add_condition(ts_attr, Sos.COND_GE, data_float[0][1])
         f.add_condition(ts_attr, Sos.COND_LE, data_float[i][1])
         Dprint(data_float[0][1], data_float[i][1])
         o = f.end()
         count = i + 1
         while o:
             count -= 1
             Dprint(o[:])
             o = f.prev()
         self.assertEqual( count, 0 )
         Dprint("--------------------------------")
コード例 #10
0
 def test_uint16_k0(self):
     a_join = self.uint16_schema.attr_by_name('a_join')
     k0 = self.uint16_schema.attr_by_name('k0')
     f = Sos.Filter(a_join)
     f.add_condition(k0, Sos.COND_GE, 4)
     f.add_condition(k0, Sos.COND_LE, 12)
     o = f.begin()
     count = 0
     while o:
         # Dprint(o[:])
         count += 1
         o = next(f)
     Dprint("Misses {0}".format(f.miss_count()))
     Dprint("Count {0}".format(count))
     self.assertTrue(f.miss_count() <= 1)
     self.assertEqual(count, 9 * 16 * 16 * 16)
     del f
コード例 #11
0
 def test_int64_k0_rev(self):
     a_join = self.int64_schema.attr_by_name('a_join')
     k0 = self.int64_schema.attr_by_name('k0')
     f = Sos.Filter(a_join)
     f.add_condition(k0, Sos.COND_LE, -4)
     f.add_condition(k0, Sos.COND_GE, -12)
     o = f.end()
     count = 0
     while o:
         # Dprint(o[:])
         count += 1
         o = f.prev()
     Dprint("Misses {0}".format(f.miss_count()))
     Dprint("Count {0}".format(count))
     self.assertTrue(f.miss_count() <= 1)
     self.assertEqual(count, 9 * 17 * 17 * 17)
     del f
コード例 #12
0
 def test_int32_k0_k1(self):
     a_join = self.int32_schema.attr_by_name('a_join')
     k0 = self.int32_schema.attr_by_name('k0')
     k1 = self.int32_schema.attr_by_name('k1')
     f = Sos.Filter(a_join)
     f.add_condition(k0, Sos.COND_LE, -4)
     f.add_condition(k0, Sos.COND_GE, -12)
     f.add_condition(k1, Sos.COND_LE, -4)
     f.add_condition(k1, Sos.COND_GE, -12)
     o = f.begin()
     count = 0
     while o:
         # Dprint(o[:])
         count += 1
         o = next(f)
     Dprint("Misses {0}".format(f.miss_count()))
     Dprint("count {0}".format(count))
     self.assertTrue(f.miss_count() <= 2)
     self.assertEqual(count, 9 * 9 * 17 * 17)
     del f
コード例 #13
0
ファイル: Analysis.py プロジェクト: opengridcomputing/SOS
 def select(self, comp_id=None, job_id=None):
     self.timestamp = self.in_schema.attr_by_name("timestamp")
     if self.index is None:
         if job_id is not None:
             self.index = "job_comp_time"
         elif comp_id is not None:
             self.index = "comp_time"
         else:
             self.index = "timestamp"
     self.filter = Sos.Filter(self.in_schema.attr_by_name(self.index))
     self.filter.add_condition(self.timestamp,
                               Sos.COND_GE,
                               self.start.strftime(self.dt_fmt))
     self.filter.add_condition(self.timestamp,
                               Sos.COND_LE,
                               self.end.strftime(self.dt_fmt))
     if job_id is not None:
         self.job_id = self.in_schema.attr_by_name("job_id")
         self.filter.add_condition(self.job_id, Sos.COND_EQ, str(job_id))
     if comp_id is not None:
         self.comp_id = self.in_schema.attr_by_name("component_id")
         self.filter.add_condition(self.comp_id, Sos.COND_EQ, str(comp_id))
コード例 #14
0
 def __query_test(self, index, job_id, comp_id, start, end, expected_count):
     join = self.schema.attr_by_name(index)
     time_a = self.schema['timestamp']
     f = Sos.Filter(join)
     if job_id is not None:
         job_a = self.schema['job_id']
         f.add_condition(job_a, Sos.COND_EQ, job_id)
     if comp_id is not None:
         comp_a = self.schema['component_id']
         f.add_condition(comp_a, Sos.COND_EQ, comp_id)
     f.add_condition(time_a, Sos.COND_GE, start)
     f.add_condition(time_a, Sos.COND_LT, end)
     o = f.begin()
     count = 0
     while o:
         # Dprint(o[:4])
         count += 1
         o = next(f)
     Dprint("Misses {0}".format(f.miss_count()))
     Dprint("Count {0}".format(count))
     self.assertTrue(f.miss_count() <= 3)
     self.assertEqual(count, expected_count)
     del f
コード例 #15
0
    def __test_next(self, attr_name):
        global key
        global data
        key, data = self.__add_data(key, data)
        attr = self.schema[attr_name]
        f = Sos.Filter(attr)
        # Iterate to the end
        count = 0
        o = f.begin()
        while o:
            d = data[count]
            v = o[:]
            self.assertEqual(d[0], v[0])
            self.assertEqual(d[1], v[1])
            count += 1
            o = next(f)
        self.assertEqual(count, len(data))

        # Add more data
        new_data = []
        key, new_data = self.__add_data( key, new_data )

        # next(f) should return the new data
        o = next(f)
        self.assertIsNotNone( o )
        count = 0
        while o:
            d = new_data[count]
            v = o[:]
            self.assertEqual(d[0], v[0])
            self.assertEqual(d[1], v[1])
            count += 1
            o = next(f)
        # we should not see any object twice
        self.assertEqual( count, len(new_data) )
        # Make the global data match the container
        data = data + new_data
コード例 #16
0
ファイル: models.py プロジェクト: nick-enoent/sosdb-ui
    def parse_query(self, request):
        self.parms = request.GET
        self.request = request
        try:
            self.parse_request(self.parms)
        except Exception as e:
            return (1, "Exception in parse_request: {0}".format(e), None)

        if not self.schema():
            return (1, "A 'schema' clause must be specified.\n", None)

        if not 'index' in self.parms:
            return (1, "An 'index' clause must be specified.\n", None)

        #
        # Open an iterator on the container
        #
        self.index_attr = None
        self.index_name = self.parms['index']
        if 'indexName' not in self.request.session:
            self.reset()
        if 'containerName' not in self.request.session:
            self.reset()
        self.schema_name = self.schema().name()
        self.index_attr = self.schema().attr_by_name(self.index_name)
        self.iter_ = self.index_attr.index().stats()
        self.filt = Sos.Filter(self.index_attr)
        if 'unique' in self.parms:
            self.unique = True
            self.filt.unique()
        else:
            self.unique = False
        self.card = self.iter_['cardinality']
        if self.unique:
            self.card = self.card - self.iter_['duplicates']

        #
        # Parse the select clause. The view_cols contains the index as it's first element.
        #
        self.view_cols = []
        if 'select' in self.parms:
            self.select = self.parms['select']
            for attr_name in self.select.split(','):
                if attr_name != self.index_name:
                    self.view_cols.append(attr_name)
        else:
            for attr in self.schema():
                if attr.name() != self.index_name:
                    self.view_cols.append(attr.name())
        #
        # Parse the where clause
        #
        #
        # A filter is an array of conditions
        #
        if 'where' in self.parms:
            where = self.parms['where']
            conds = where.split(',')
            for cond in conds:
                tokens = cond.split(':')
                if len(tokens) < 3:
                    return (1, "Invalid where clause '{0}', "
                            "valid syntax is attr:cmp_str:value".format(cond),
                            None)
                attr_name = tokens[0]
                attr = self.schema().attr_by_name(attr_name)
                if not attr:
                    return (1, "The attribute {0} was not found "
                            "in the schema {1}".format(attr_name,
                                                       self.schema().name()),
                            None)
                sos_cmp = sos_filt_cond[tokens[1]]
                value_str = None
                tokens[2] = tokens[2].split('"')[1]
                for s in tokens[2:]:
                    if value_str:
                        value_str = value_str + ':' + s
                    else:
                        value_str = s
                self.filt.add_condition(attr, sos_cmp, int(tokens[2]))

        obj = None
        if self.start == 0:
            self.reset()
            obj = self.filt.begin()
            skip = self.start
        elif self.start + self.count >= self.card:
            self.reset()
            obj = self.filt.end()
            skip = self.card % self.count
            while obj and skip > 0:
                obj = self.filt.prev()
                skip = skip - 1
        else:
            self.filt.set_pos(str(self.request.session['pos']))
            skip = self.start - self.request.session['recordNo']

            obj = self.filt.obj()
            while obj and skip != 0:
                if skip > 0:
                    obj = next(self.filt)
                    skip = skip - 1
                else:
                    obj = self.filt.prev()
                    skip = skip + 1
        pos = self.filt.get_pos()
        self.request.session['pos'] = pos
        self.request.session['recordNo'] = self.start
        return (0, None, obj)
コード例 #17
0
 def setUp(self):
     self.filt = Sos.Filter(self.schema['job_time_comp'])