def test_get_create_table_command(self):
schema = t1s.Ta1ResultsSchema()
desired_F2A_sql_cmd = "".join([
"CREATE TABLE IF NOT EXISTS %s (" % t1s.F2A_TABLENAME,
"%s INTEGER NOT NULL, " % t1s.F2A_FQID,
"%s INTEGER NOT NULL, " % t1s.F2A_AQID,
"FOREIGN KEY (%s) REFERENCES %s (ROWID), " %
(t1s.F2A_FQID, t1s.DBF_TABLENAME),
"FOREIGN KEY (%s) REFERENCES %s (ROWID))" %
(t1s.F2A_AQID, t1s.DBA_TABLENAME)
])
self.assertEquals(schema.get_create_table_command(t1s.F2A_TABLENAME),
desired_F2A_sql_cmd)
def _store_latency_by_fieldtype_graph(self):
"""Stores the latency as a function of fieldtype graph."""
# find all of the naming and reference strings:
caption = "Type %s Queries (%s)" % (self._inp[
t1s.DBF_CAT], self._inp.test_db.get_short_database_name())
tag = self.get_tag(self._inp)
graph_path = self.get_img_path(self._inp, "byfieldtype")
# find the data and create the graph:
constraint_list = (self._config.get_constraint_list() +
self._inp.get_constraint_list())
categories = self._config.results_db.get_unique_query_values(
simple_fields=[(t1s.DBP_TABLENAME, t1s.DBP_SELECTIONCOLS)],
constraint_list=constraint_list,
atomic_fields_and_functions=[
(t1s.DBA_FIELDTYPE,
t1s.Ta1ResultsSchema().get_complex_function(
t1s.DBA_TABLENAME, t1s.DBA_FIELDTYPE))
])
(x_label,
y_label) = (self._config.var_nummatches + " = # new matching records",
self._config.var_ql + " = query latency (s)")
datasets = []
for (selection_cols, field_type) in categories:
this_constraint_list = constraint_list + [
(t1s.DBP_TABLENAME, t1s.DBP_SELECTIONCOLS, selection_cols),
(t1s.DBA_TABLENAME, t1s.DBA_FIELDTYPE, field_type)
]
[x_values, y_values] = self._config.results_db.get_query_values(
[(t1s.DBP_TABLENAME, t1s.DBP_NUMNEWRETURNEDRECORDS),
(t1s.DBP_TABLENAME, t1s.DBP_QUERYLATENCY)],
constraint_list=this_constraint_list)
try:
inputs = [x_values]
outputs = y_values
function = regression.regress(
function_to_regress=self._config.ql_all_ftr,
outputs=outputs,
inputs=inputs)
except regression.BadRegressionInputError:
function = None
datasets.append(
(x_values, y_values,
"SELECT %s on %ss" % (selection_cols, field_type), function))
graph = graphing.graph2d(plot_name="",
datasets=datasets,
x_label=x_label,
y_label=y_label)
self.write(graph_path, graph)
graph_image = latex_classes.LatexImage(caption, tag, graph_path)
self._outp["latency_by_fieldtype_graph"] = graph_image.get_string()
def get_sub_query_types():
"""returns a string represenging the sub-query types."""
sub_query_types = self._config.results_db.get_unique_query_values(
atomic_fields_and_functions=[
(t1s.DBA_CAT,
t1s.Ta1ResultsSchema().get_complex_function(
t1s.DBA_TABLENAME, t1s.DBA_CAT, is_list=True))
],
constraint_list=self._config.get_constraint_list() +
[(t1s.DBF_TABLENAME, t1s.DBF_CAT, cat_string),
(t1s.DBF_TABLENAME, t1s.DBF_SUBCAT, subcat_string)])
sub_query_types = list(set(sum(sub_query_types, ())))
sub_query_types_string = ",".join(sub_query_types)
return sub_query_types_string
def _store_latency_by_keywordlen_graph(self):
"""Stores the latency as a function of keywordlen graph."""
# find all of the naming and reference strings:
caption = "Type %s Queries (%s)" % (self._inp[
t1s.DBF_CAT], self._inp.test_db.get_short_database_name())
tag = self.get_tag(inp=self._inp)
graph_path = self.get_img_path(inp=self._inp, aux="bykeyword")
# find the data and create the graphs, etc:
keywordlen_numrecords_max = self._config.keywordlen_numrecords_max[
self._inp["cat_number"]]
keywordlen_numrecords_min = self._config.keywordlen_numrecords_min[
self._inp["cat_number"]]
self._outp["keywordlen_numrecords_max"] = keywordlen_numrecords_max
self._outp["keywordlen_numrecords_min"] = keywordlen_numrecords_min
this_constraint_list = (self._config.get_constraint_list() +
self._inp.get_constraint_list())
this_non_standard_constraint_list = [
(t1s.DBP_TABLENAME, t1s.DBP_NUMNEWRETURNEDRECORDS,
"%s.%s<" + str(keywordlen_numrecords_max)),
(t1s.DBP_TABLENAME, t1s.DBP_NUMNEWRETURNEDRECORDS,
"%s.%s>" + str(keywordlen_numrecords_min))
]
these_atomic_fields_and_functions = [
(t1s.DBA_KEYWORDLEN, t1s.Ta1ResultsSchema().get_complex_function(
t1s.DBA_TABLENAME, t1s.DBA_KEYWORDLEN))
]
categories = self._config.results_db.get_unique_query_values(
[(t1s.DBP_TABLENAME, t1s.DBP_SELECTIONCOLS)],
atomic_fields_and_functions=these_atomic_fields_and_functions,
constraint_list=this_constraint_list,
non_standard_constraint_list=this_non_standard_constraint_list)
y_label = (self._config.var_ql + " = query latency (s)")
inputs = []
for (selection_cols, keyword_len) in categories:
data = self._config.results_db.get_query_values(
[(t1s.DBP_TABLENAME, t1s.DBP_QUERYLATENCY)],
constraint_list=this_constraint_list +
[(t1s.DBP_TABLENAME, t1s.DBP_SELECTIONCOLS, selection_cols),
(t1s.DBA_TABLENAME, t1s.DBA_KEYWORDLEN, keyword_len)],
non_standard_constraint_list=this_non_standard_constraint_list)
label = "SELECT %s, keyword legnth %s" % (selection_cols,
str(keyword_len))
inputs.append((label, data))
if inputs:
graph = graphing.box_plot("", inputs)
self.write(graph_path, graph)
graphimage = latex_classes.LatexImage(caption, tag, graph_path)
self._outp["latency_by_keywordlen_graph"] = graphimage.get_string()
def test_get_unique_atomic_query_values_with_simple_constraint(self):
this_constraint_list = [(t1s.DBF_TABLENAME, t1s.DBF_CAT, "Eq")]
# atomic unique values will not appear in a tuple this time, because
# the function passed in specifies that only one value is expected.
schema = t1s.Ta1ResultsSchema()
expected_unique_fieldtype_values = set(["string", "integer"])
unique_fieldtype_values = set(
self.database.get_unique_query_values(
[],
constraint_list=this_constraint_list,
atomic_fields_and_functions=[
(t1s.DBA_FIELDTYPE,
schema.get_complex_function(t1s.DBA_TABLENAME,
t1s.DBA_FIELDTYPE))
]))
self.assertEqual(unique_fieldtype_values,
expected_unique_fieldtype_values)
def _discover_querytypes(self):
"""Populates the present_categories and atomic_present_categories
attributes."""
constraint_list = [(t1s.DBP_TABLENAME, t1s.DBP_PERFORMERNAME,
self.config.performername)]
simple_fields = [(t1s.DBF_TABLENAME, t1s.DBF_CAT),
(t1s.DBF_TABLENAME, t1s.DBF_SUBCAT),
(t1s.DBF_TABLENAME, t1s.DBF_SUBSUBCAT),
(t1s.DBF_TABLENAME, t1s.DBF_NUMRECORDS),
(t1s.DBF_TABLENAME, t1s.DBF_RECORDSIZE)]
present_cat_strings = self.config.results_db.get_unique_query_values(
simple_fields=simple_fields,
constraint_list=constraint_list)
self.present_cats = []
self.atomic_present_cats = []
for (cat_str, subcat_str, subsubcat_str,
dbnr, dbrs) in present_cat_strings:
cat = t1s.CATEGORIES.value_to_number[cat_str]
if subcat_str:
subcat = t1s.SUBCATEGORIES[cat].value_to_number[subcat_str]
else:
subcat = None
if subsubcat_str:
subsubcat = int(subsubcat_str)
else:
subsubcat = None
self.present_cats.append((cat, subcat, subsubcat, dbnr, dbrs))
if cat in t1s.ATOMIC_CATEGORIES:
these_atomic_fields_and_functions = [
(t1s.DBA_FIELDTYPE,
t1s.Ta1ResultsSchema().get_complex_function(
t1s.DBA_TABLENAME, t1s.DBA_FIELDTYPE))]
this_constraint_list = constraint_list + [
(t1s.DBF_TABLENAME, t1s.DBF_CAT, cat_str),
(t1s.DBF_TABLENAME, t1s.DBF_SUBCAT, subcat_str),
(t1s.DBF_TABLENAME, t1s.DBF_SUBSUBCAT, subsubcat_str),
(t1s.DBF_TABLENAME, t1s.DBF_NUMRECORDS, dbnr),
(t1s.DBF_TABLENAME, t1s.DBF_RECORDSIZE, dbrs)]
present_fieldtype_strings = self.config.results_db.get_unique_query_values(
atomic_fields_and_functions=these_atomic_fields_and_functions,
constraint_list=this_constraint_list)
for fieldtype_str in present_fieldtype_strings:
fieldtype = t1s.TEST_FIELD_TYPES.value_to_number[
fieldtype_str]
self.atomic_present_cats.append(
(cat, subcat, subsubcat, dbnr, dbrs, fieldtype))
def test_process_to_database(self):
schema = t1s.Ta1ResultsSchema()
tablename = t1s.DBF_TABLENAME
field_simple = t1s.DBF_FQID
value_simple = 7
field_list_int = t1s.DBF_MATCHINGRECORDIDS
value_list_int_db = "1|2|34"
value_list_int_use = [1, 2, 34]
field_list_str = t1s.DBF_MATCHINGRECORDHASHES
value_list_str_db = "hash1|hash2|hash3"
value_list_str_use = ["hash1", "hash2", "hash3"]
self.assertEqual(
schema.process_to_database(tablename, field_simple, value_simple),
value_simple)
self.assertEqual(
schema.process_to_database(tablename, field_list_int,
value_list_int_use), value_list_int_db)
self.assertEqual(
schema.process_to_database(tablename, field_list_str,
value_list_str_use), value_list_str_db)
def _get_parameters(self, selection_cols):
"""Returns parameters for the 3d graph."""
parameters = {}
parameters["z_label"] = (
self._config.var_rangesize + " = range size")
# find the data:
this_constraint_list = (
self._config.get_constraint_list() +
self._inp.get_constraint_list() + [
(t1s.DBP_TABLENAME, t1s.DBP_SELECTIONCOLS, selection_cols)])
these_atomic_fields_and_functions = [
(t1s.DBA_RANGE,
t1s.Ta1ResultsSchema().get_complex_function(t1s.DBA_TABLENAME,
t1s.DBA_RANGE))]
parameters["values"] = self._config.results_db.get_query_values(
[(t1s.DBP_TABLENAME, t1s.DBP_NUMNEWRETURNEDRECORDS),
(t1s.DBP_TABLENAME, t1s.DBP_QUERYLATENCY)],
constraint_list=this_constraint_list,
atomic_fields_and_functions=these_atomic_fields_and_functions)
parameters["ftr"] = self._config.ql_p2_ftr
return parameters
def test_get_unique_atomic_query_values_without_constraint(self):
# atomic unique values will appear in a tuple, because there
# is always the chance that multiple atomic values are associated with
# a single performer query id.
schema = t1s.Ta1ResultsSchema()
expected_unique_fieldtype_values = set([
tuple(["string"]),
tuple(["integer"]),
tuple(["integer", "string"]),
tuple(["string", "integer"])
])
unique_fieldtype_values = set(
self.database.get_unique_query_values(
[],
atomic_fields_and_functions=[
(t1s.DBA_FIELDTYPE,
schema.get_complex_function(t1s.DBA_TABLENAME,
t1s.DBA_FIELDTYPE,
is_list=True))
]))
self.assertEqual(unique_fieldtype_values,
expected_unique_fieldtype_values)
def setUp(self):
self.database = results_database.ResultsDB(
db_path=":memory:", schema=t1s.Ta1ResultsSchema())
set_up_static_db(self.database)
def __init__(self, db_path):
"""
Initializes the database with a location (db_path) and a logger.
"""
schema = t1s.Ta1ResultsSchema()
super(Ta1ResultsDB, self).__init__(db_path, schema)
