def test_get_create_table_command(self): schema = t1s.Ta1ResultsSchema() desired_F2A_sql_cmd = "".join([ "CREATE TABLE IF NOT EXISTS %s (" % t1s.F2A_TABLENAME, "%s INTEGER NOT NULL, " % t1s.F2A_FQID, "%s INTEGER NOT NULL, " % t1s.F2A_AQID, "FOREIGN KEY (%s) REFERENCES %s (ROWID), " % (t1s.F2A_FQID, t1s.DBF_TABLENAME), "FOREIGN KEY (%s) REFERENCES %s (ROWID))" % (t1s.F2A_AQID, t1s.DBA_TABLENAME) ]) self.assertEquals(schema.get_create_table_command(t1s.F2A_TABLENAME), desired_F2A_sql_cmd)
def _store_latency_by_fieldtype_graph(self): """Stores the latency as a function of fieldtype graph.""" # find all of the naming and reference strings: caption = "Type %s Queries (%s)" % (self._inp[ t1s.DBF_CAT], self._inp.test_db.get_short_database_name()) tag = self.get_tag(self._inp) graph_path = self.get_img_path(self._inp, "byfieldtype") # find the data and create the graph: constraint_list = (self._config.get_constraint_list() + self._inp.get_constraint_list()) categories = self._config.results_db.get_unique_query_values( simple_fields=[(t1s.DBP_TABLENAME, t1s.DBP_SELECTIONCOLS)], constraint_list=constraint_list, atomic_fields_and_functions=[ (t1s.DBA_FIELDTYPE, t1s.Ta1ResultsSchema().get_complex_function( t1s.DBA_TABLENAME, t1s.DBA_FIELDTYPE)) ]) (x_label, y_label) = (self._config.var_nummatches + " = # new matching records", self._config.var_ql + " = query latency (s)") datasets = [] for (selection_cols, field_type) in categories: this_constraint_list = constraint_list + [ (t1s.DBP_TABLENAME, t1s.DBP_SELECTIONCOLS, selection_cols), (t1s.DBA_TABLENAME, t1s.DBA_FIELDTYPE, field_type) ] [x_values, y_values] = self._config.results_db.get_query_values( [(t1s.DBP_TABLENAME, t1s.DBP_NUMNEWRETURNEDRECORDS), (t1s.DBP_TABLENAME, t1s.DBP_QUERYLATENCY)], constraint_list=this_constraint_list) try: inputs = [x_values] outputs = y_values function = regression.regress( function_to_regress=self._config.ql_all_ftr, outputs=outputs, inputs=inputs) except regression.BadRegressionInputError: function = None datasets.append( (x_values, y_values, "SELECT %s on %ss" % (selection_cols, field_type), function)) graph = graphing.graph2d(plot_name="", datasets=datasets, x_label=x_label, y_label=y_label) self.write(graph_path, graph) graph_image = latex_classes.LatexImage(caption, tag, graph_path) self._outp["latency_by_fieldtype_graph"] = graph_image.get_string()
def get_sub_query_types(): """returns a string represenging the sub-query types.""" sub_query_types = self._config.results_db.get_unique_query_values( atomic_fields_and_functions=[ (t1s.DBA_CAT, t1s.Ta1ResultsSchema().get_complex_function( t1s.DBA_TABLENAME, t1s.DBA_CAT, is_list=True)) ], constraint_list=self._config.get_constraint_list() + [(t1s.DBF_TABLENAME, t1s.DBF_CAT, cat_string), (t1s.DBF_TABLENAME, t1s.DBF_SUBCAT, subcat_string)]) sub_query_types = list(set(sum(sub_query_types, ()))) sub_query_types_string = ",".join(sub_query_types) return sub_query_types_string
def _store_latency_by_keywordlen_graph(self): """Stores the latency as a function of keywordlen graph.""" # find all of the naming and reference strings: caption = "Type %s Queries (%s)" % (self._inp[ t1s.DBF_CAT], self._inp.test_db.get_short_database_name()) tag = self.get_tag(inp=self._inp) graph_path = self.get_img_path(inp=self._inp, aux="bykeyword") # find the data and create the graphs, etc: keywordlen_numrecords_max = self._config.keywordlen_numrecords_max[ self._inp["cat_number"]] keywordlen_numrecords_min = self._config.keywordlen_numrecords_min[ self._inp["cat_number"]] self._outp["keywordlen_numrecords_max"] = keywordlen_numrecords_max self._outp["keywordlen_numrecords_min"] = keywordlen_numrecords_min this_constraint_list = (self._config.get_constraint_list() + self._inp.get_constraint_list()) this_non_standard_constraint_list = [ (t1s.DBP_TABLENAME, t1s.DBP_NUMNEWRETURNEDRECORDS, "%s.%s<" + str(keywordlen_numrecords_max)), (t1s.DBP_TABLENAME, t1s.DBP_NUMNEWRETURNEDRECORDS, "%s.%s>" + str(keywordlen_numrecords_min)) ] these_atomic_fields_and_functions = [ (t1s.DBA_KEYWORDLEN, t1s.Ta1ResultsSchema().get_complex_function( t1s.DBA_TABLENAME, t1s.DBA_KEYWORDLEN)) ] categories = self._config.results_db.get_unique_query_values( [(t1s.DBP_TABLENAME, t1s.DBP_SELECTIONCOLS)], atomic_fields_and_functions=these_atomic_fields_and_functions, constraint_list=this_constraint_list, non_standard_constraint_list=this_non_standard_constraint_list) y_label = (self._config.var_ql + " = query latency (s)") inputs = [] for (selection_cols, keyword_len) in categories: data = self._config.results_db.get_query_values( [(t1s.DBP_TABLENAME, t1s.DBP_QUERYLATENCY)], constraint_list=this_constraint_list + [(t1s.DBP_TABLENAME, t1s.DBP_SELECTIONCOLS, selection_cols), (t1s.DBA_TABLENAME, t1s.DBA_KEYWORDLEN, keyword_len)], non_standard_constraint_list=this_non_standard_constraint_list) label = "SELECT %s, keyword legnth %s" % (selection_cols, str(keyword_len)) inputs.append((label, data)) if inputs: graph = graphing.box_plot("", inputs) self.write(graph_path, graph) graphimage = latex_classes.LatexImage(caption, tag, graph_path) self._outp["latency_by_keywordlen_graph"] = graphimage.get_string()
def test_get_unique_atomic_query_values_with_simple_constraint(self): this_constraint_list = [(t1s.DBF_TABLENAME, t1s.DBF_CAT, "Eq")] # atomic unique values will not appear in a tuple this time, because # the function passed in specifies that only one value is expected. schema = t1s.Ta1ResultsSchema() expected_unique_fieldtype_values = set(["string", "integer"]) unique_fieldtype_values = set( self.database.get_unique_query_values( [], constraint_list=this_constraint_list, atomic_fields_and_functions=[ (t1s.DBA_FIELDTYPE, schema.get_complex_function(t1s.DBA_TABLENAME, t1s.DBA_FIELDTYPE)) ])) self.assertEqual(unique_fieldtype_values, expected_unique_fieldtype_values)
def _discover_querytypes(self): """Populates the present_categories and atomic_present_categories attributes.""" constraint_list = [(t1s.DBP_TABLENAME, t1s.DBP_PERFORMERNAME, self.config.performername)] simple_fields = [(t1s.DBF_TABLENAME, t1s.DBF_CAT), (t1s.DBF_TABLENAME, t1s.DBF_SUBCAT), (t1s.DBF_TABLENAME, t1s.DBF_SUBSUBCAT), (t1s.DBF_TABLENAME, t1s.DBF_NUMRECORDS), (t1s.DBF_TABLENAME, t1s.DBF_RECORDSIZE)] present_cat_strings = self.config.results_db.get_unique_query_values( simple_fields=simple_fields, constraint_list=constraint_list) self.present_cats = [] self.atomic_present_cats = [] for (cat_str, subcat_str, subsubcat_str, dbnr, dbrs) in present_cat_strings: cat = t1s.CATEGORIES.value_to_number[cat_str] if subcat_str: subcat = t1s.SUBCATEGORIES[cat].value_to_number[subcat_str] else: subcat = None if subsubcat_str: subsubcat = int(subsubcat_str) else: subsubcat = None self.present_cats.append((cat, subcat, subsubcat, dbnr, dbrs)) if cat in t1s.ATOMIC_CATEGORIES: these_atomic_fields_and_functions = [ (t1s.DBA_FIELDTYPE, t1s.Ta1ResultsSchema().get_complex_function( t1s.DBA_TABLENAME, t1s.DBA_FIELDTYPE))] this_constraint_list = constraint_list + [ (t1s.DBF_TABLENAME, t1s.DBF_CAT, cat_str), (t1s.DBF_TABLENAME, t1s.DBF_SUBCAT, subcat_str), (t1s.DBF_TABLENAME, t1s.DBF_SUBSUBCAT, subsubcat_str), (t1s.DBF_TABLENAME, t1s.DBF_NUMRECORDS, dbnr), (t1s.DBF_TABLENAME, t1s.DBF_RECORDSIZE, dbrs)] present_fieldtype_strings = self.config.results_db.get_unique_query_values( atomic_fields_and_functions=these_atomic_fields_and_functions, constraint_list=this_constraint_list) for fieldtype_str in present_fieldtype_strings: fieldtype = t1s.TEST_FIELD_TYPES.value_to_number[ fieldtype_str] self.atomic_present_cats.append( (cat, subcat, subsubcat, dbnr, dbrs, fieldtype))
def test_process_to_database(self): schema = t1s.Ta1ResultsSchema() tablename = t1s.DBF_TABLENAME field_simple = t1s.DBF_FQID value_simple = 7 field_list_int = t1s.DBF_MATCHINGRECORDIDS value_list_int_db = "1|2|34" value_list_int_use = [1, 2, 34] field_list_str = t1s.DBF_MATCHINGRECORDHASHES value_list_str_db = "hash1|hash2|hash3" value_list_str_use = ["hash1", "hash2", "hash3"] self.assertEqual( schema.process_to_database(tablename, field_simple, value_simple), value_simple) self.assertEqual( schema.process_to_database(tablename, field_list_int, value_list_int_use), value_list_int_db) self.assertEqual( schema.process_to_database(tablename, field_list_str, value_list_str_use), value_list_str_db)
def _get_parameters(self, selection_cols): """Returns parameters for the 3d graph.""" parameters = {} parameters["z_label"] = ( self._config.var_rangesize + " = range size") # find the data: this_constraint_list = ( self._config.get_constraint_list() + self._inp.get_constraint_list() + [ (t1s.DBP_TABLENAME, t1s.DBP_SELECTIONCOLS, selection_cols)]) these_atomic_fields_and_functions = [ (t1s.DBA_RANGE, t1s.Ta1ResultsSchema().get_complex_function(t1s.DBA_TABLENAME, t1s.DBA_RANGE))] parameters["values"] = self._config.results_db.get_query_values( [(t1s.DBP_TABLENAME, t1s.DBP_NUMNEWRETURNEDRECORDS), (t1s.DBP_TABLENAME, t1s.DBP_QUERYLATENCY)], constraint_list=this_constraint_list, atomic_fields_and_functions=these_atomic_fields_and_functions) parameters["ftr"] = self._config.ql_p2_ftr return parameters
def test_get_unique_atomic_query_values_without_constraint(self): # atomic unique values will appear in a tuple, because there # is always the chance that multiple atomic values are associated with # a single performer query id. schema = t1s.Ta1ResultsSchema() expected_unique_fieldtype_values = set([ tuple(["string"]), tuple(["integer"]), tuple(["integer", "string"]), tuple(["string", "integer"]) ]) unique_fieldtype_values = set( self.database.get_unique_query_values( [], atomic_fields_and_functions=[ (t1s.DBA_FIELDTYPE, schema.get_complex_function(t1s.DBA_TABLENAME, t1s.DBA_FIELDTYPE, is_list=True)) ])) self.assertEqual(unique_fieldtype_values, expected_unique_fieldtype_values)
def setUp(self): self.database = results_database.ResultsDB( db_path=":memory:", schema=t1s.Ta1ResultsSchema()) set_up_static_db(self.database)
def __init__(self, db_path): """ Initializes the database with a location (db_path) and a logger. """ schema = t1s.Ta1ResultsSchema() super(Ta1ResultsDB, self).__init__(db_path, schema)