def setUpClass(cls):
cls.setUpDb("schema_test_cont")
cls.schema = Sos.Schema()
cls.schema.from_template('schema_attr_test', attr_list)
cls.schema.add(cls.db)
cls.types = [
Sos.TYPE_UINT64,
Sos.TYPE_INT16,
Sos.TYPE_INT32,
Sos.TYPE_INT64,
Sos.TYPE_UINT16,
Sos.TYPE_UINT32,
Sos.TYPE_UINT64,
Sos.TYPE_FLOAT,
Sos.TYPE_DOUBLE,
Sos.TYPE_LONG_DOUBLE,
Sos.TYPE_TIMESTAMP,
Sos.TYPE_STRUCT,
Sos.TYPE_BYTE_ARRAY,
Sos.TYPE_CHAR_ARRAY,
Sos.TYPE_INT16_ARRAY,
Sos.TYPE_INT32_ARRAY,
Sos.TYPE_INT64_ARRAY,
Sos.TYPE_UINT16_ARRAY,
Sos.TYPE_UINT32_ARRAY,
Sos.TYPE_UINT64_ARRAY,
Sos.TYPE_FLOAT_ARRAY,
Sos.TYPE_DOUBLE_ARRAY,
Sos.TYPE_LONG_DOUBLE_ARRAY
]
def setUpClass(cls):
cls.setUpDb("obj_set_get_cont")
cls.schema = Sos.Schema()
cls.schema.from_template('obj_set_get', [
{ "name" : "int16", "type" : "int16" },
{ "name" : "int32", "type" : "int32" },
{ "name" : "int64", "type" : "int64" },
{ "name" : "uint16", "type" : "uint16" },
{ "name" : "uint32", "type" : "uint32" },
{ "name" : "uint64", "type" : "uint64" },
{ "name" : "float", "type" : "float" },
{ "name" : "double", "type" : "double" },
{ "name" : "long_double", "type" : "long_double" },
{ "name" : "timestamp", "type" : "timestamp" },
{ "name" : "struct", "type" : "struct", "size" : 24 },
{ "name" : "byte_array", "type" : "byte_array" },
{ "name" : "char_array", "type" : "char_array" },
{ "name" : "int16_array", "type" : "int16_array" },
{ "name" : "int32_array", "type" : "int32_array" },
{ "name" : "int64_array", "type" : "int64_array" },
{ "name" : "uint16_array", "type" : "uint16_array" },
{ "name" : "uint32_array", "type" : "uint32_array" },
{ "name" : "uint64_array", "type" : "uint64_array" },
{ "name" : "float_array", "type" : "float_array" },
{ "name" : "double_array", "type" : "double_array" },
{ "name" : "long_double_array", "type" : "long_double_array" }
])
cls.schema.add(cls.db)
def setUpClass(cls):
cls.setUpDb("timestamp_test_cont")
cls.schema = Sos.Schema()
cls.schema.from_template('timestamp_test',
[
{ "name" : "timestamp", "type" : "timestamp", "index" : {} },
{ "name" : "timestr", "type" : "char_array" }
])
cls.schema.add(cls.db)
def setUpClass(cls):
cls.setUpDb('append_data_test_cont')
cls.schema = Sos.Schema()
cls.schema.from_template('append_data_test',
[
{ "name" : "int32", "type" : "int32", "index" : {} },
{ "name" : "string", "type" : "char_array" }
])
cls.schema.add(cls.db)
def setUpClass(cls):
cls.setUpDb('query_test2_cont')
cls.schema = Sos.Schema()
cls.schema.from_template('query_test2', [{
"name": "int16",
"type": "int16",
"index": {}
}])
cls.schema.add(cls.db)
def setUpClass(cls):
cls.setUpDb("h2htbl_cont")
cls.schema = Sos.Schema()
cls.schema.from_template('test_h2htbl',
[ { "name" : "tkn_id", "type" : "uint64",
"index" : { "type" : "H2HTBL" } },
{ "name" : "tkn_count", "type" : "uint64", },
{ "name" : "tkn_text", "type" : "char_array",
"index" : { "type" : "h2htbl" } },
])
cls.schema.add(cls.db)
def setUpClass(cls):
cls.setUpDb("pattern_test_cont")
cls.schema = Sos.Schema()
cls.schema.from_template('test_pattern',
[ { "name" : "ts", "type" : "timestamp", "index" : {}},
{ "name" : "ptn_id", "type" : "uint64",
"index" : {}, },
{ "name" : "ts_ptn_key", "type" : "join",
"join_attrs" : [ "ts", "ptn_id" ],
"index" : {}}
])
cls.schema.add(cls.db)
def setUpClass(cls):
cls.setUpDb("join_test_str_cont")
cls.schema = Sos.Schema()
cls.schema.from_template('test_str',
[ { "name" : "a_1", "type" : "string" },
{ "name" : "a_2", "type" : "string" },
{ "name" : "a_3", "type" : "string" },
{ "name" : "a_join", "type" : "join",
"join_attrs" : [ "a_1", "a_2", "a_3" ],
"index" : {}}
])
cls.schema.add(cls.db)
def setUpClass(cls):
cls.setUpDb("min_max_test_cont")
cls.schema = Sos.Schema()
cls.schema.from_template('test_min_max',
[
{ "name" : "uint32", "type" : "uint32", "index" : {} },
{ "name" : "int32", "type" : "int32", "index" : {} },
{ "name" : "uint64", "type" : "uint64", "index" : {} },
{ "name" : "int64", "type" : "int64", "index" : {} },
{ "name" : "double", "type" : "double", "index" : {} },
{ "name" : "string", "type" : "string", "index" : {} },
])
cls.schema.add(cls.db)
def setUpClass(cls):
cls.setUpDb("update_test_cont")
cls.schema = Sos.Schema()
cls.schema.from_template('update_test', [
{ "name" : "int16", "type" : "int16" },
{ "name" : "int32", "type" : "int32" },
{ "name" : "int64", "type" : "int64" },
{ "name" : "uint16", "type" : "uint16" },
{ "name" : "uint32", "type" : "uint32" },
{ "name" : "uint64", "type" : "uint64" },
{ "name" : "float", "type" : "float" },
{ "name" : "double", "type" : "double" },
{ "name" : "key", "type" : "uint64", "index" : {} }
])
cls.schema.add(cls.db)
def setUpClass(cls):
cls.setUpDb("delete_test_cont")
cls.schema = Sos.Schema()
cls.schema.from_template('test_delete', [{
"name": "ts",
"type": "timestamp"
}, {
"name": "id",
"type": "uint64"
}, {
"name": "key",
"type": "join",
"join_attrs": ["ts", "id"],
"index": {}
}])
cls.schema.add(cls.db)
def setUpClass(cls):
cls.setUpDb("metric_gui_join")
t = time.time()
cls.start = t + 60
cls.end = t + 120
cls.schema = Sos.Schema()
cls.schema.from_template(
'metric_set',
[{
"name": "timestamp",
"type": "timestamp",
"index": {}
}, {
"name": "component_id",
"type": "uint64",
"index": {}
}, {
"name": "job_id",
"type": "uint64",
"index": {}
}, {
"name": "app_id",
"type": "uint64"
}, {
"name": "metric0",
"type": "uint64"
}, {
"name": "comp_time",
"type": "join",
"join_attrs": ["component_id", "timestamp"],
"index": {}
}, {
"name": "job_comp_time",
"type": "join",
"join_attrs": ["job_id", "component_id", "timestamp"],
"index": {}
}, {
"name": "job_time_comp",
"type": "join",
"join_attrs": ["job_id", "timestamp", "component_id"],
"index": {}
}])
cls.schema.add(cls.db)
def set_output(self, path):
self.out_cont = Sos.Container()
try:
self.out_cont.open(path, Sos.PERM_RW)
except:
self.out_cont.create(path)
self.out_cont.open(path, Sos.PERM_RW)
self.out_cont.part_create("RESULTS")
part = self.out_cont.part_by_name("RESULTS")
part.state_set("primary")
self.out_schema = self.out_cont.schema_by_name(self.out_schema_name)
if not self.out_schema:
if self.out_schema_template is None:
raise ValueError("A schema template must be provided.". \
format(self.out_schema_name))
self.out_schema = Sos.Schema()
self.out_schema.from_template(self.out_schema_name, self.out_schema_template)
self.out_schema.add(self.out_cont)
def setUpClass(cls):
cls.setUpDb('join_test_3x_u64_cont')
cls.schema = Sos.Schema()
cls.schema.from_template('3x_u64', [{
"name": "a_1",
"type": "uint64"
}, {
"name": "a_2",
"type": "uint64"
}, {
"name": "a_3",
"type": "uint64"
}, {
"name": "join_key",
"type": "join",
"join_attrs": ["a_1", "a_2", "a_3"],
"index": {}
}])
cls.schema.add(cls.db)
def setUpClass(cls):
cls.setUpDb("array_test_cont")
cls.schema = Sos.Schema()
cls.schema.from_template('array_test', [{
"name": "byte_array",
"type": "byte_array"
}, {
"name": "char_array",
"type": "char_array",
"index": {}
}, {
"name": "int16_array",
"type": "int16_array"
}, {
"name": "int32_array",
"type": "int32_array"
}, {
"name": "int64_array",
"type": "int64_array"
}, {
"name": "uint16_array",
"type": "uint16_array"
}, {
"name": "uint32_array",
"type": "uint32_array"
}, {
"name": "uint64_array",
"type": "uint64_array"
}, {
"name": "float_array",
"type": "float_array"
}, {
"name": "double_array",
"type": "double_array"
}, {
"name": "long_double_array",
"type": "long_double_array"
}])
cls.schema.add(cls.db)
def setUpClass(cls):
cls.setUpDb('pos_test_cont')
cls.schema = Sos.Schema()
cls.schema_name = '3x_u64'
cls.schema.from_template(
cls.schema_name,
[{
"name": "job_id",
"type": "uint64"
}, {
"name": "timestamp",
"type": "timestamp"
}, {
"name": "component_id",
"type": "uint64"
}, {
"name": "job_time_comp",
"type": "join",
"join_attrs": ["job_id", "timestamp", "component_id"],
"index": {}
}])
cls.schema.add(cls.db)
def setUpClass(cls):
cls.setUpDb("join_test_filter_seek")
cls.uint64_schema = Sos.Schema()
cls.uint64_schema.from_template(
'uint64_four_way_join', [{
"name": "k0",
"type": "uint64"
}, {
"name": "k1",
"type": "uint64"
}, {
"name": "k2",
"type": "uint64"
}, {
"name": "k3",
"type": "uint64"
}, {
"name": "a_join",
"type": "join",
"join_attrs": ["k0", "k1", "k2", "k3"],
"index": {}
}])
cls.uint64_schema.add(cls.db)
cls.int64_schema = Sos.Schema()
cls.int64_schema.from_template(
'int64_four_way_join', [{
"name": "k0",
"type": "int64"
}, {
"name": "k1",
"type": "int64"
}, {
"name": "k2",
"type": "int64"
}, {
"name": "k3",
"type": "int64"
}, {
"name": "a_join",
"type": "join",
"join_attrs": ["k0", "k1", "k2", "k3"],
"index": {}
}])
cls.int64_schema.add(cls.db)
cls.uint32_schema = Sos.Schema()
cls.uint32_schema.from_template(
'uint32_four_way_join', [{
"name": "k0",
"type": "uint32"
}, {
"name": "k1",
"type": "uint32"
}, {
"name": "k2",
"type": "uint32"
}, {
"name": "k3",
"type": "uint32"
}, {
"name": "a_join",
"type": "join",
"join_attrs": ["k0", "k1", "k2", "k3"],
"index": {}
}])
cls.uint32_schema.add(cls.db)
cls.int32_schema = Sos.Schema()
cls.int32_schema.from_template(
'int32_four_way_join', [{
"name": "k0",
"type": "int32"
}, {
"name": "k1",
"type": "int32"
}, {
"name": "k2",
"type": "int32"
}, {
"name": "k3",
"type": "int32"
}, {
"name": "a_join",
"type": "join",
"join_attrs": ["k0", "k1", "k2", "k3"],
"index": {}
}])
cls.int32_schema.add(cls.db)
cls.uint16_schema = Sos.Schema()
cls.uint16_schema.from_template(
'uint16_four_way_join', [{
"name": "k0",
"type": "uint16"
}, {
"name": "k1",
"type": "uint16"
}, {
"name": "k2",
"type": "uint16"
}, {
"name": "k3",
"type": "uint16"
}, {
"name": "a_join",
"type": "join",
"join_attrs": ["k0", "k1", "k2", "k3"],
"index": {}
}])
cls.uint16_schema.add(cls.db)
cls.int16_schema = Sos.Schema()
cls.int16_schema.from_template(
'int16_four_way_join', [{
"name": "k0",
"type": "int16"
}, {
"name": "k1",
"type": "int16"
}, {
"name": "k2",
"type": "int16"
}, {
"name": "k3",
"type": "int16"
}, {
"name": "a_join",
"type": "join",
"join_attrs": ["k0", "k1", "k2", "k3"],
"index": {}
}])
cls.int16_schema.add(cls.db)
def setUpClass(cls):
cls.setUpDb("key_test_cont")
cls.schema = Sos.Schema()
cls.schema.from_template('key_test', [
{
"name": "int16",
"type": "int16",
"index": {}
},
{
"name": "int32",
"type": "int32",
"index": {}
},
{
"name": "int64",
"type": "int64",
"index": {}
},
{
"name": "uint16",
"type": "uint16",
"index": {}
},
{
"name": "uint32",
"type": "uint32",
"index": {}
},
{
"name": "uint64",
"type": "uint64",
"index": {}
},
{
"name": "float",
"type": "float",
"index": {}
},
{
"name": "double",
"type": "double",
"index": {}
},
{
"name": "timestamp",
"type": "timestamp",
"index": {}
},
{
"name": "struct",
"type": "struct",
"size": 16,
"index": {}
},
{
"name": "string",
"type": "char_array",
"index": {}
},
{
"name": "byte_array",
"type": "byte_array",
"index": {}
},
{
"name": "int16_array",
"type": "int16_array",
"index": {}
},
{
"name": "int32_array",
"type": "int32_array",
"index": {}
},
{
"name": "int64_array",
"type": "int64_array",
"index": {}
},
{
"name": "uint16_array",
"type": "uint16_array",
"index": {}
},
{
"name": "uint32_array",
"type": "uint32_array",
"index": {}
},
{
"name": "uint64_array",
"type": "uint64_array",
"index": {}
},
{
"name": "float_array",
"type": "float_array",
"index": {}
},
{
"name": "double_array",
"type": "double_array",
"index": {}
},
])
cls.schema.add(cls.db)
def test_05_many_schema(self):
for i in range(0, 10 * 1024):
schema = Sos.Schema()
schema.from_template('schema_{0}'.format(i), attr_list)
schema.add(self.db)
self.assertTrue(1 == 1)
def setUpClass(cls):
cls.setUpDb('query_test_cont')
cls.schema1 = Sos.Schema()
cls.schema1.from_template(
'query_test', [{
"name": "int16",
"type": "int16",
"index": {}
}, {
"name": "int32",
"type": "int32",
"index": {}
}, {
"name": "int64",
"type": "int64",
"index": {}
}, {
"name": "uint16",
"type": "uint16",
"index": {}
}, {
"name": "uint32",
"type": "uint32",
"index": {}
}, {
"name": "uint64",
"type": "uint64",
"index": {}
}, {
"name": "float",
"type": "float",
"index": {}
}, {
"name": "double",
"type": "double",
"index": {}
}, {
"name": "timestamp",
"type": "timestamp",
"index": {}
}, {
"name": "string",
"type": "char_array",
"index": {}
}, {
"name": "byte_array",
"type": "byte_array",
"index": {}
}, {
"name": "int16_array",
"type": "int16_array",
"index": {}
}, {
"name": "int32_array",
"type": "int32_array",
"index": {}
}, {
"name": "int64_array",
"type": "int64_array",
"index": {}
}, {
"name": "uint16_array",
"type": "uint16_array",
"index": {}
}, {
"name": "uint32_array",
"type": "uint32_array",
"index": {}
}, {
"name": "uint64_array",
"type": "uint64_array",
"index": {}
}, {
"name": "float_array",
"type": "float_array",
"index": {}
}, {
"name": "double_array",
"type": "double_array",
"index": {}
}, {
"name": "int32_no_idx",
"type": "int32"
}, {
"name": "int16_int32_",
"type": "join",
"join_attrs": ["int16", "int32"],
"index": {}
}, {
"name": "int16_int32_int64",
"type": "join",
"join_attrs": ["int16", "int32", "int64"],
"index": {}
}])
cls.schema1.add(cls.db)
cls.schema2 = Sos.Schema()
cls.schema2.from_template('query_test_2', [
{
"name": "int32_2",
"type": "int32",
"index": {}
},
{
"name": "timestamp",
"type": "timestamp",
"index": {}
},
])
cls.schema2.add(cls.db)
cls.query = Sos.Query(cls.db)
# Add a partition
db.part_create("ROOT")
part = db.part_by_name("ROOT")
part.state_set("primary")
# Create the test schema
template = [{
"name": "key",
"type": "uint64",
"index": {}
}, {
"name": "order",
"type": "uint64"
}]
schema = Sos.Schema()
schema.from_template("inf_sup", template)
# Add the schema to the container
schema.add(db)
# Add data to test the inf/sup functions
data = [[1, 1], [1, 2], [1, 3], [2, 4], [2, 5], [2, 6], [4, 7], [4, 8], [4, 9],
[5, 10], [5, 11], [5, 12]]
for entry in data:
obj = schema.alloc()
obj[0] = entry[0]
obj[1] = entry[1]
obj.index_add()
def setUpClass(cls):
# purge existing store
shutil.rmtree(cls.STORE_PATH, ignore_errors=True)
try:
# create new store
cls.cont = sos.Container()
cls.cont.create(cls.STORE_PATH)
cls.cont.open(cls.STORE_PATH)
# new partition
cls.cont.part_create(cls.PART_NAME)
part = cls.cont.part_by_name(cls.PART_NAME)
part.state_set("PRIMARY")
del part
except Exception as e:
raise StandardError(
"The test container could not be created.".format(
cls.STORE_PATH))
# new schema
cls.schema = schema = sos.Schema()
schema.from_template(
cls.SCHEMA_NAME,
[
{
"name": "i32",
"type": "INT32",
"index": {
"type": cls.IDX_TYPE,
"key": "INT32",
"args": cls.IDX_ARG,
}
},
{
# {uint32, uint32, uint64}
"name": "struct",
"type": "STRUCT",
"size": 16,
"index": {
"type": cls.IDX_TYPE,
"key": "MEMCMP",
"args": cls.IDX_ARG,
}
},
])
cls.attr_key_half_next = [
attr_key_half_next_INT32,
attr_key_half_next_STRUCT,
]
cls.attr_key_half_prev = [
attr_key_half_prev_INT32,
attr_key_half_prev_STRUCT,
]
schema.add(cls.cont)
cls.input_data = [ (i, struct.pack("!LLLL", i, i, i, i)) \
for i in range(10, 500, 10) \
for _ in range(REPEAT) ]
# data
for d in cls.input_data:
obj = schema.alloc()
obj[:] = d
obj.index_add()
def add_schema(self, template):
"""Add a schema to the data source
parameters:
- template -- A dictionary that defines the name, attributes
and indices in the schema.
A schema template is a Python dictionary:
{
"name" : '<SCHEMA-NAME>',
"attrs" : [ <ATTR-DEFINITION>, ]
}
The "name" entry specifies the name of the schema and must be
unique within the container.
The "attrs" entry is a list of ATTR-DEFITION, each of which is
a Python dictionary.
{
"name" : "<string>",
"type" : "<string>",
"index" : { INDEX-DEFINITION }
}
The "name" entry specifies the name of the attribute in the
schema and must be unique within the schema.
The "type" attribute is a string specifying a type names
as follows:
- "INT16"
- "INT32"
- "INT64"
- "UINT16"
- "UINT32"
- "UINT64"
- "FLOAT"
- "DOUBLE",
- "LONG_DOUBLE"
- "TIMESTAMP"
- "STRUCT"
- "JOIN"
- "BYTE_ARRAY"
- "CHAR_ARRAY"
- "STRING"
- "INT16_ARRAY"
- "INT32_ARRAY"
- "INT64_ARRAY"
- "UINT16_ARRAY"
- "UINT32_ARRAY"
- "UINT64_ARRAY"
- "FLOAT_ARRAY"
- "DOUBLE_ARRAY"
- "LONG_DOUBLE_ARRAY"
Type names are not case sensitive.
If the type name is "JOIN", a "join_attrs" list is specified
that indicates which attributes are going to be combined
together to form the value of the "JOIN" attribute. Attributes
that are to be joined must have been previously defined in the
template.
A "JOIN" attributes does not consume any space in the object
as there values are computed on the fly from the other values
in the object. A "JOIN" attribute is used as a key values for
an index in order to order object data in the container.
The "index" entry is optional but if present, the attribute
value becomes the key to an index that will be associated with
the attribute. The contents of the dictionary object argument
to the index attribute specifies optional features of the
index. If it is empty, i.e. {}, the defaults are used for the
index.
Example Schema Template:
a_new_schema = \
{ "name" : "a_new_schema",
"attrs" : [
{ "name" : "timestamp", "type" : "timestamp", "index" : {} },
{ "name" : "component_id", "type" : "uint64" },
{ "name" : "flits", "type" : "double" },
{ "name" : "stalls", "type" : "double" },
{ "name" : "comp_time", "type" : "join",
"join_attrs" : [ "component_id", "timestamp" ],
"index" : {} }
]
}
In this example, the "timestamp", and "comp_time" attributes
will be indexed.
schema = src.add_schema(a_new_schema)
"""
if self.cont is None:
raise ValueError("The container is not open")
schema = Sos.Schema()
schema = schema.from_template(template['name'], template['attrs'])
schema.add(self.cont)
def setUpClass(cls):
cls.setUpDb('filter_test_cont')
cls.schema = Sos.Schema()
cls.schema.from_template('filter_test', [
{
"name": "int16",
"type": "int16",
"index": {}
},
{
"name": "int32",
"type": "int32",
"index": {}
},
{
"name": "int64",
"type": "int64",
"index": {}
},
{
"name": "uint16",
"type": "uint16",
"index": {}
},
{
"name": "uint32",
"type": "uint32",
"index": {}
},
{
"name": "uint64",
"type": "uint64",
"index": {}
},
{
"name": "float",
"type": "float",
"index": {}
},
{
"name": "double",
"type": "double",
"index": {}
},
{
"name": "timestamp",
"type": "timestamp",
"index": {}
},
{
"name": "string",
"type": "char_array",
"index": {}
},
{
"name": "byte_array",
"type": "byte_array",
"index": {}
},
{
"name": "int16_array",
"type": "int16_array",
"index": {}
},
{
"name": "int32_array",
"type": "int32_array",
"index": {}
},
{
"name": "int64_array",
"type": "int64_array",
"index": {}
},
{
"name": "uint16_array",
"type": "uint16_array",
"index": {}
},
{
"name": "uint32_array",
"type": "uint32_array",
"index": {}
},
{
"name": "uint64_array",
"type": "uint64_array",
"index": {}
},
{
"name": "float_array",
"type": "float_array",
"index": {}
},
{
"name": "double_array",
"type": "double_array",
"index": {}
},
{
"name": "join_i16_i32_i64",
"type": "join",
"join_attrs": ["int16", "int32", "int64"],
"index": {}
},
{
"name": "join_u16_u32_u64",
"type": "join",
"join_attrs": ["uint16", "uint32", "uint64"],
"index": {}
},
{
"name": "join_i16_u32_i64",
"type": "join",
"join_attrs": ["int16", "uint32", "int64"],
"index": {}
},
{
"name": "join_i16_i32_u64",
"type": "join",
"join_attrs": ["int16", "int32", "uint64"],
"index": {}
},
{
"name": "join_i64_double",
"type": "join",
"join_attrs": ["int64", "double"],
"index": {}
},
{
"name": "join_u64_timestamp",
"type": "join",
"join_attrs": ["uint64", "timestamp"],
"index": {}
},
])
cls.schema.add(cls.db)