def setup_query_parameters(config): now = config.day + dt.timedelta(1) now_str = now.strftime('%Y-%m-%d') yesterday = config.day yesterday_str = yesterday.strftime('%Y-%m-%d') logger.debug("config.day = %s; now = %s; yesterday = %s", config.day, now, yesterday) prod_phrase = '' try: if config.product != '': if ',' in config.product: prod_list = [x.strip() for x in config.product.split(',')] prod_phrase = ("and r.product in ('%s')" % "','".join(prod_list)) else: prod_phrase = "and r.product = '%s'" % config.product except Exception: util.reportExceptionAndContinue(logger) ver_phrase = '' try: if config.version != '': if ',' in config.product: ver_list = [x.strip() for x in config.version.split(',')] ver_phrase = ("and r.version in ('%s')" % "','".join(ver_list)) else: ver_phrase = "and r.version = '%s'" % config.version except Exception: util.reportExceptionAndContinue(logger) return util.DotDict({ 'now_str': now_str, 'yesterday_str': yesterday_str, 'prod_phrase': prod_phrase, 'ver_phrase': ver_phrase })
def test_parse_arguments_with_class_validators(self): class NumberConverter(object): def clean(self, value): conv = {'one': 1, 'two': 2, 'three': 3} try: return conv[value] except KeyError: raise ValueError('No idea?!') # Define a set of filters with some types being non-trivial types # but instead a custom validator. filters = [ ("param1", 0, NumberConverter()), ] arguments = { "param1": "one", } params_exp = util.DotDict() params_exp.param1 = 1 params = external_common.parse_arguments(filters, arguments, modern=True) eq_(params, params_exp) # note that a ValueError becomes a BadArgumentError arguments = { "param1": "will cause a ValueError in NumberConverter.clean", } assert_raises(BadArgumentError, external_common.parse_arguments, filters, arguments, modern=True)
def get_dummy_context(self): """Create a dummy config object to use when testing.""" context = util.DotDict({ 'database_class': ConnectionContext, 'database_hostname': 'somewhere', 'database_port': '8888', 'database_name': 'somename', 'database_username': '******', 'database_password': '******', }) return context
def test_parse_arguments_old_way(self): """Test external_common.parse_arguments(). """ filters = [("param1", "default", ["list", "str"]), ("param2", None, "int"), ("param3", ["list", "of", 4, "values"], ["list", "str"])] arguments = {"param1": "value1", "unknown": 12345} params_exp = util.DotDict() params_exp.param1 = ["value1"] params_exp.param2 = None params_exp.param3 = ["list", "of", "4", "values"] params = external_common.parse_arguments( filters, arguments, modern=False, ) eq_(params, params_exp)
def test_parse_arguments(self): """Test external_common.parse_arguments(). """ filters = [ ("param1", "default", [str]), ("param2", None, int), ("param3", ["some", "default", "list"], [str]), ("param4", ["list", "of", 4, "values"], [str]), ("param5", None, bool), ("param6", None, datetime.date), ("param7", None, datetime.date), ("param8", None, datetime.datetime), ("param9", None, [str]), ] arguments = { "param1": "value1", "unknown": 12345, "param5": "true", "param7": datetime.date(2016, 2, 9).isoformat(), "param8": datetime.datetime(2016, 2, 9).isoformat(), # note the 'param9' is deliberately not specified. } params_exp = util.DotDict() params_exp.param1 = ["value1"] params_exp.param2 = None params_exp.param3 = ['some', 'default', 'list'] params_exp.param4 = ["list", "of", "4", "values"] params_exp.param5 = True params_exp.param6 = None params_exp.param7 = datetime.date(2016, 2, 9) params_exp.param8 = datetime.datetime(2016, 2, 9).replace(tzinfo=isodate.UTC) params_exp.param9 = None params = external_common.parse_arguments(filters, arguments, modern=True) for key in params: eq_(params[key], params_exp[key], '{}: {!r} != {!r}'.format(key, params[key], params_exp[key])) eq_(params, params_exp)
def get_dummy_context(self): """Create a dummy config object to use when testing.""" context = util.DotDict({ 'database_class': ConnectionContext, 'database_hostname': 'somewhere', 'database_port': '8888', 'database_name': 'somename', 'database_username': '******', 'database_password': '******', }) context.platforms = ( { "id": "windows", "name": "Windows NT" }, { "id": "linux", "name": "Linux" } ) context.non_release_channels = ['beta', 'aurora', 'nightly'] context.restricted_channels = ['beta'] return context
def test_build_reports_sql_from(self): """Test PostgreSQLBase.build_reports_sql_from().""" pgbase = self.get_instance() params = util.DotDict() params.report_process = "" # ..................................................................... # Test 1: no specific parameter sql_exp = "FROM reports r" sql = pgbase.build_reports_sql_from(params) eq_(sql, sql_exp) # ..................................................................... # Test 2: with a plugin params.report_process = "plugin" sql_exp = "FROM reports r JOIN plugins_reports ON " \ "plugins_reports.report_id = r.id JOIN plugins ON " \ "plugins_reports.plugin_id = plugins.id" sql = pgbase.build_reports_sql_from(params) sql = " ".join(sql.split()) # squeeze all \s, \r, \t... eq_(sql, sql_exp)
def test_build_version_where(self): """Test PostgreSQLBase.build_reports_sql_version_where().""" config = self.get_dummy_context() pgbase = self.get_instance() params = util.DotDict() params["versions_info"] = { "Firefox:12.0a1": { "version_string": "12.0a1", "product_name": "Firefox", "major_version": "12.0", "release_channel": "Nightly", "build_id": ["20120101123456"] }, "Fennec:11.0": { "version_string": "11.0", "product_name": "Fennec", "major_version": None, "release_channel": None, "build_id": None }, "Firefox:13.0b": { "version_string": "13.0b", "product_name": "Firefox", "major_version": "13.0", "release_channel": "beta", "build_id": ["20120101123456", "20120101098765"] }, "WaterWolf:1.0a1": { "version_string": "1.0a1", "product_name": "WaterWolf", "major_version": "1.0a1", "release_channel": "nightly-water", "build_id": None }, "WaterWolf:2.0": { "version_string": "2.0", "product_name": "WaterWolf", "major_version": "2.0", "release_channel": None, "build_id": None } } # test 1 params["versions"] = ["Firefox", "13.0b"] key = "Firefox:13.0b" x = 0 sql_params = { "version0": "Firefox", "version1": "13.0b" } sql_params_exp = { "version0": "Firefox", "version1": "13.0b" } version_where = [] version_where_exp = ( "(" "r.release_channel ILIKE 'beta'" " AND r.build IN ('20120101123456', '20120101098765')" " AND r.product=%(version0)s" " AND r.version=%(version1)s" ")" ) version_where = pgbase.build_version_where( "Firefox", "13.0b", x, sql_params, params["versions_info"][key], config, ) eq_(version_where, version_where_exp) eq_(sql_params, sql_params_exp) # test 2, verify release channels get added as expected params["versions"] = ["WaterWolf", "1.0a1"] key = "WaterWolf:1.0a1" x = 0 sql_params = { "version0": "WaterWolf", "version1": "1.0a1" } sql_params_exp = { "version0": "WaterWolf", "version1": "1.0a1" } version_where = [] version_where_exp = ( "(" "r.release_channel ILIKE 'nightly-water'" " AND r.product=%(version0)s" " AND r.version=%(version1)s" ")" ) version_where = pgbase.build_version_where( "WaterWolf", "1.0a1", x, sql_params, params["versions_info"][key], config, ) eq_(version_where, version_where_exp) eq_(sql_params, sql_params_exp) # test 3, what if a release channel is "null" params["versions"] = ["WaterWolf", "2.0"] key = "WaterWolf:2.0" x = 0 sql_params = { "version0": "WaterWolf", "version1": "2.0" } sql_params_exp = { "version0": "WaterWolf", "version1": "2.0" } version_where_exp = ( "(" "r.product=%(version0)s" " AND r.version=%(version1)s" ")" ) version_where = pgbase.build_version_where( "WaterWolf", "2.0", x, sql_params, params["versions_info"][key], config, ) eq_(version_where, version_where_exp) eq_(sql_params, sql_params_exp)
def test_build_reports_sql_where(self): """ Test PostgreSQLBase.build_reports_sql_where().""" config = self.get_dummy_context() pgbase = self.get_instance() params = search_common.get_parameters({}) # Get default search params default_params = util.DotDict(params.copy()) sql_params = {} # ..................................................................... # Test 1: default values for parameters sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \ "%(to_date)s" sql_params_exp = { "from_date": params.from_date, "to_date": params.to_date } (sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params, config) sql = " ".join(sql.split()) # squeeze all \s, \r, \t... eq_(sql, sql_exp) eq_(sql_params, sql_params_exp) # ..................................................................... # Test 2: terms and search_mode = is_exactly sql_params = {} params.terms = "signature" params.search_mode = "is_exactly" sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \ "%(to_date)s AND r.signature=%(term)s" sql_params_exp = { "from_date": params.from_date, "to_date": params.to_date, "term": params.terms } (sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params, config) sql = " ".join(sql.split()) # squeeze all \s, \r, \t... eq_(sql, sql_exp) eq_(sql_params, sql_params_exp) # ..................................................................... # Test 3: terms and search_mode != is_exactly sql_params = {} params.terms = "signature%" params.search_mode = "starts_with" sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \ "%(to_date)s AND r.signature LIKE %(term)s" sql_params_exp = { "from_date": params.from_date, "to_date": params.to_date, "term": params.terms } (sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params, config) sql = " ".join(sql.split()) # squeeze all \s, \r, \t... eq_(sql, sql_exp) eq_(sql_params, sql_params_exp) # ..................................................................... # Test 4: products sql_params = {} params.terms = default_params.terms params.search_mode = default_params.search_mode params.products = ["Firefox", "Fennec"] sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \ "%(to_date)s AND (r.product=%(product0)s OR " \ "r.product=%(product1)s)" sql_params_exp = { "from_date": params.from_date, "to_date": params.to_date, "product0": "Firefox", "product1": "Fennec" } (sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params, config) sql = " ".join(sql.split()) # squeeze all \s, \r, \t... eq_(sql, sql_exp) eq_(sql_params, sql_params_exp) # ..................................................................... # Test 5: os sql_params = {} params.products = default_params.products params.os = ["Windows"] sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \ "%(to_date)s AND (r.os_name=%(os0)s)" sql_params_exp = { "from_date": params.from_date, "to_date": params.to_date, "os0": "Windows" } (sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params, config) sql = " ".join(sql.split()) # squeeze all \s, \r, \t... eq_(sql, sql_exp) eq_(sql_params, sql_params_exp) # ..................................................................... # Test 6: build_ids sql_params = {} params.os = default_params.os params.build_ids = ["20120101123456"] sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \ "%(to_date)s AND (r.build=%(build0)s)" sql_params_exp = { "from_date": params.from_date, "to_date": params.to_date, "build0": "20120101123456" } (sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params, config) sql = " ".join(sql.split()) # squeeze all \s, \r, \t... eq_(sql, sql_exp) eq_(sql_params, sql_params_exp) # ..................................................................... # Test 7: reasons sql_params = {} params.build_ids = default_params.build_ids params.reasons = ["EXCEPTION", "OVERFLOW"] sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \ "%(to_date)s AND (r.reason=%(reason0)s OR " \ "r.reason=%(reason1)s)" sql_params_exp = { "from_date": params.from_date, "to_date": params.to_date, "reason0": "EXCEPTION", "reason1": "OVERFLOW" } (sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params, config) sql = " ".join(sql.split()) # squeeze all \s, \r, \t... eq_(sql, sql_exp) eq_(sql_params, sql_params_exp) # ..................................................................... # Test 8: report_type sql_params = {} params.reasons = default_params.reasons params.report_type = "crash" sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \ "%(to_date)s AND r.hangid IS NULL" sql_params_exp = { "from_date": params.from_date, "to_date": params.to_date } (sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params, config) sql = " ".join(sql.split()) # squeeze all \s, \r, \t... eq_(sql, sql_exp) eq_(sql_params, sql_params_exp) # ..................................................................... # Test 9: versions sql_params = {} params.report_type = default_params.report_type params.versions = ["Firefox", "12.0a1", "Fennec", "11.0", "Firefox", "13.0b"] params.versions_info = { "Firefox:12.0a1": { "version_string": "12.0a1", "product_name": "Firefox", "major_version": "12.0", "release_channel": "Nightly", "build_id": ["20120101123456"], "is_rapid_beta": False, "from_rapid_beta": False, "from_beta_version": "Firefox:12.0a1", }, "Fennec:11.0": { "version_string": "11.0", "product_name": "Fennec", "major_version": None, "release_channel": None, "build_id": None, "is_rapid_beta": False, "from_rapid_beta": False, "from_beta_version": "Fennec:11.0", }, "Firefox:13.0b1": { "version_string": "13.0b1", "product_name": "Firefox", "major_version": "13.0", "release_channel": "Beta", "build_id": ["20120101123456", "20120101098765"], "is_rapid_beta": False, "from_rapid_beta": True, "from_beta_version": "Firefox:13.0b", }, "Firefox:13.0b": { "version_string": "13.0b", "product_name": "Firefox", "major_version": "13.0b", "release_channel": "Beta", "build_id": None, "is_rapid_beta": True, "from_rapid_beta": True, "from_beta_version": "Firefox:13.0b", } } sql_exp = """ WHERE r.date_processed BETWEEN %(from_date)s AND %(to_date)s AND ((r.release_channel ILIKE 'nightly' AND r.product=%(version0)s AND r.version=%(version1)s) OR (r.product=%(version2)s AND r.version=%(version3)s) OR (r.product=%(version4)s AND r.version=%(version5)s) OR (r.product=%(version6)s AND r.version=%(version7)s) OR (r.release_channel ILIKE 'beta' AND r.build IN ('20120101123456', '20120101098765') AND r.product=%(version8)s AND r.version=%(version9)s) OR (r.product=%(version10)s AND r.version=%(version11)s)) """ sql_params_exp = { "from_date": params.from_date, "to_date": params.to_date, "version0": "Firefox", "version1": "12.0a1", "version2": "Firefox", "version3": "12.0a1", "version4": "Fennec", "version5": "11.0", "version6": "Fennec", "version7": "11.0", "version8": "Firefox", "version9": "13.0b1", "version10": "Firefox", "version11": "13.0b", } (sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params, config) # squeeze all \s, \r, \t... sql = " ".join(sql.split()) sql_exp = " ".join(sql_exp.split()) eq_(sql, sql_exp) eq_(sql_params, sql_params_exp) # ..................................................................... # Test 10: report_process = plugin sql_params = {} params.versions = default_params.versions params.versions_infos = None params.report_process = "plugin" sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \ "%(to_date)s AND r.process_type = 'plugin' AND " \ "plugins_reports.date_processed BETWEEN " \ "%(from_date)s AND %(to_date)s" sql_params_exp = { "from_date": params.from_date, "to_date": params.to_date } (sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params, config) sql = " ".join(sql.split()) # squeeze all \s, \r, \t... eq_(sql, sql_exp) eq_(sql_params, sql_params_exp) # ..................................................................... # Test 11: report_process != plugin sql_params = {} params.report_process = "content" sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \ "%(to_date)s AND r.process_type = 'content'" sql_params_exp = { "from_date": params.from_date, "to_date": params.to_date } (sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params, config) sql = " ".join(sql.split()) # squeeze all \s, \r, \t... eq_(sql, sql_exp) eq_(sql_params, sql_params_exp) # ..................................................................... # Test 12: plugins sql_params = {} params.report_process = "plugin" params.plugin_terms = "plugin_name" params.plugin_search_mode = "is_exactly" params.plugin_in = ["name"] sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \ "%(to_date)s AND r.process_type = 'plugin' AND " \ "plugins_reports.date_processed BETWEEN " \ "%(from_date)s AND %(to_date)s AND " \ "(plugins.name=%(plugin_term)s)" sql_params_exp = { "from_date": params.from_date, "to_date": params.to_date, "plugin_term": params.plugin_terms } (sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params, config) sql = " ".join(sql.split()) # squeeze all \s, \r, \t... eq_(sql, sql_exp) eq_(sql_params, sql_params_exp)
def setUp(self): """ Populate product_info table with fake data """ super(IntegrationTestTCBS, self).setUp() cursor = self.connection.cursor() self.now = datetimeutil.utc_now() now = self.now.date() yesterday = now - datetime.timedelta(days=1) lastweek = now - datetime.timedelta(days=7) self.params = util.DotDict() self.params.startDate = lastweek self.params.product = 'Firefox' self.params.version = '8.0' self.params.limit = 100 self.params.to_date = yesterday self.params.date_range_type = None self.params.duration = datetime.timedelta(7) self.params.os = None self.params.crash_type = 'all' self.params.logger = mock.Mock() cursor.execute(""" INSERT INTO signatures (signature_id, signature, first_report) VALUES (1, 'Fake Signature #1', '%(first_date)s'), (2, 'Fake Signature #2', '%(first_date)s'), (3, 'Fake Signature #3', '%(first_date)s'); """ % {'first_date': lastweek}) cursor.execute(""" INSERT INTO products (product_name, sort, rapid_release_version, release_name) VALUES ('Firefox', 1, '8.0', 'firefox'), ('Fennec', 2, '11.0', 'mobile'); """) cursor.execute(""" INSERT INTO release_channels (release_channel, sort) VALUES ('Release', 1); """) cursor.execute(""" INSERT INTO product_versions (product_version_id, product_name, major_version, release_version, version_string, build_date, sunset_date, featured_version, build_type) VALUES ( 1, 'Firefox', '8.0', '8.0', '8.0', '%(start_date)s', '%(end_date)s', False, 'Release' ), ( 2, 'Fennec', '11.0', '11.0', '11.0.1', '%(start_date)s', '%(end_date)s', False, 'Release' ); """ % {'start_date': self.now - datetime.timedelta(days=8), 'end_date': self.now}) cursor.execute(""" INSERT INTO signature_products_rollup (signature_id, product_name, ver_count, version_list) VALUES ( 1, 'Firefox', 2, '{plugin1, plugin2}' ), ( 2, 'Firefox', 6, '{plugin1, plugin2, plugin3, plugin4, plugin5, plugin6}' ), ( 3, 'Fennec', 3, '{plugin1, plugin2}' ); """) cursor.execute(""" INSERT INTO tcbs ( signature_id, report_date, product_version_id, process_type, release_channel, report_count, win_count, mac_count, lin_count, hang_count, is_gc_count ) VALUES (1, '%(lastweek)s', 1, 'type', 'Release', 14, 12, 1, 1, 0, 1), (2, '%(now)s', 1, 'crash', 'Release', 3, 1, 1, 1, 0, 1), (1, '%(now)s', 1, 'hang', 'Release', 5, 0, 0, 5, 5, 10), (2, '%(lastweek)s', 1, 'crash', 'Release', 10, 7, 2, 1, 0, 3); """ % {'now': self.now, 'lastweek': self.now - datetime.timedelta(days=8)}) self.connection.commit()