def setup_query_parameters(config):
now = config.day + dt.timedelta(1)
now_str = now.strftime('%Y-%m-%d')
yesterday = config.day
yesterday_str = yesterday.strftime('%Y-%m-%d')
logger.debug("config.day = %s; now = %s; yesterday = %s", config.day, now,
yesterday)
prod_phrase = ''
try:
if config.product != '':
if ',' in config.product:
prod_list = [x.strip() for x in config.product.split(',')]
prod_phrase = ("and r.product in ('%s')" %
"','".join(prod_list))
else:
prod_phrase = "and r.product = '%s'" % config.product
except Exception:
util.reportExceptionAndContinue(logger)
ver_phrase = ''
try:
if config.version != '':
if ',' in config.product:
ver_list = [x.strip() for x in config.version.split(',')]
ver_phrase = ("and r.version in ('%s')" % "','".join(ver_list))
else:
ver_phrase = "and r.version = '%s'" % config.version
except Exception:
util.reportExceptionAndContinue(logger)
return util.DotDict({
'now_str': now_str,
'yesterday_str': yesterday_str,
'prod_phrase': prod_phrase,
'ver_phrase': ver_phrase
})
def test_parse_arguments_with_class_validators(self):
class NumberConverter(object):
def clean(self, value):
conv = {'one': 1, 'two': 2, 'three': 3}
try:
return conv[value]
except KeyError:
raise ValueError('No idea?!')
# Define a set of filters with some types being non-trivial types
# but instead a custom validator.
filters = [
("param1", 0, NumberConverter()),
]
arguments = {
"param1": "one",
}
params_exp = util.DotDict()
params_exp.param1 = 1
params = external_common.parse_arguments(filters,
arguments,
modern=True)
eq_(params, params_exp)
# note that a ValueError becomes a BadArgumentError
arguments = {
"param1": "will cause a ValueError in NumberConverter.clean",
}
assert_raises(BadArgumentError,
external_common.parse_arguments,
filters,
arguments,
modern=True)
def get_dummy_context(self):
"""Create a dummy config object to use when testing."""
context = util.DotDict({
'database_class': ConnectionContext,
'database_hostname': 'somewhere',
'database_port': '8888',
'database_name': 'somename',
'database_username': '******',
'database_password': '******',
})
return context
def test_parse_arguments_old_way(self):
"""Test external_common.parse_arguments(). """
filters = [("param1", "default", ["list", "str"]),
("param2", None, "int"),
("param3", ["list", "of", 4, "values"], ["list", "str"])]
arguments = {"param1": "value1", "unknown": 12345}
params_exp = util.DotDict()
params_exp.param1 = ["value1"]
params_exp.param2 = None
params_exp.param3 = ["list", "of", "4", "values"]
params = external_common.parse_arguments(
filters,
arguments,
modern=False,
)
eq_(params, params_exp)
def test_parse_arguments(self):
"""Test external_common.parse_arguments(). """
filters = [
("param1", "default", [str]),
("param2", None, int),
("param3", ["some", "default", "list"], [str]),
("param4", ["list", "of", 4, "values"], [str]),
("param5", None, bool),
("param6", None, datetime.date),
("param7", None, datetime.date),
("param8", None, datetime.datetime),
("param9", None, [str]),
]
arguments = {
"param1": "value1",
"unknown": 12345,
"param5": "true",
"param7": datetime.date(2016, 2, 9).isoformat(),
"param8": datetime.datetime(2016, 2, 9).isoformat(),
# note the 'param9' is deliberately not specified.
}
params_exp = util.DotDict()
params_exp.param1 = ["value1"]
params_exp.param2 = None
params_exp.param3 = ['some', 'default', 'list']
params_exp.param4 = ["list", "of", "4", "values"]
params_exp.param5 = True
params_exp.param6 = None
params_exp.param7 = datetime.date(2016, 2, 9)
params_exp.param8 = datetime.datetime(2016, 2,
9).replace(tzinfo=isodate.UTC)
params_exp.param9 = None
params = external_common.parse_arguments(filters,
arguments,
modern=True)
for key in params:
eq_(params[key], params_exp[key],
'{}: {!r} != {!r}'.format(key, params[key], params_exp[key]))
eq_(params, params_exp)
def get_dummy_context(self):
"""Create a dummy config object to use when testing."""
context = util.DotDict({
'database_class': ConnectionContext,
'database_hostname': 'somewhere',
'database_port': '8888',
'database_name': 'somename',
'database_username': '******',
'database_password': '******',
})
context.platforms = (
{
"id": "windows",
"name": "Windows NT"
},
{
"id": "linux",
"name": "Linux"
}
)
context.non_release_channels = ['beta', 'aurora', 'nightly']
context.restricted_channels = ['beta']
return context
def test_build_reports_sql_from(self):
"""Test PostgreSQLBase.build_reports_sql_from()."""
pgbase = self.get_instance()
params = util.DotDict()
params.report_process = ""
# .....................................................................
# Test 1: no specific parameter
sql_exp = "FROM reports r"
sql = pgbase.build_reports_sql_from(params)
eq_(sql, sql_exp)
# .....................................................................
# Test 2: with a plugin
params.report_process = "plugin"
sql_exp = "FROM reports r JOIN plugins_reports ON " \
"plugins_reports.report_id = r.id JOIN plugins ON " \
"plugins_reports.plugin_id = plugins.id"
sql = pgbase.build_reports_sql_from(params)
sql = " ".join(sql.split()) # squeeze all \s, \r, \t...
eq_(sql, sql_exp)
def test_build_version_where(self):
"""Test PostgreSQLBase.build_reports_sql_version_where()."""
config = self.get_dummy_context()
pgbase = self.get_instance()
params = util.DotDict()
params["versions_info"] = {
"Firefox:12.0a1": {
"version_string": "12.0a1",
"product_name": "Firefox",
"major_version": "12.0",
"release_channel": "Nightly",
"build_id": ["20120101123456"]
},
"Fennec:11.0": {
"version_string": "11.0",
"product_name": "Fennec",
"major_version": None,
"release_channel": None,
"build_id": None
},
"Firefox:13.0b": {
"version_string": "13.0b",
"product_name": "Firefox",
"major_version": "13.0",
"release_channel": "beta",
"build_id": ["20120101123456", "20120101098765"]
},
"WaterWolf:1.0a1": {
"version_string": "1.0a1",
"product_name": "WaterWolf",
"major_version": "1.0a1",
"release_channel": "nightly-water",
"build_id": None
},
"WaterWolf:2.0": {
"version_string": "2.0",
"product_name": "WaterWolf",
"major_version": "2.0",
"release_channel": None,
"build_id": None
}
}
# test 1
params["versions"] = ["Firefox", "13.0b"]
key = "Firefox:13.0b"
x = 0
sql_params = {
"version0": "Firefox",
"version1": "13.0b"
}
sql_params_exp = {
"version0": "Firefox",
"version1": "13.0b"
}
version_where = []
version_where_exp = (
"("
"r.release_channel ILIKE 'beta'"
" AND r.build IN ('20120101123456', '20120101098765')"
" AND r.product=%(version0)s"
" AND r.version=%(version1)s"
")"
)
version_where = pgbase.build_version_where(
"Firefox",
"13.0b",
x,
sql_params,
params["versions_info"][key],
config,
)
eq_(version_where, version_where_exp)
eq_(sql_params, sql_params_exp)
# test 2, verify release channels get added as expected
params["versions"] = ["WaterWolf", "1.0a1"]
key = "WaterWolf:1.0a1"
x = 0
sql_params = {
"version0": "WaterWolf",
"version1": "1.0a1"
}
sql_params_exp = {
"version0": "WaterWolf",
"version1": "1.0a1"
}
version_where = []
version_where_exp = (
"("
"r.release_channel ILIKE 'nightly-water'"
" AND r.product=%(version0)s"
" AND r.version=%(version1)s"
")"
)
version_where = pgbase.build_version_where(
"WaterWolf",
"1.0a1",
x,
sql_params,
params["versions_info"][key],
config,
)
eq_(version_where, version_where_exp)
eq_(sql_params, sql_params_exp)
# test 3, what if a release channel is "null"
params["versions"] = ["WaterWolf", "2.0"]
key = "WaterWolf:2.0"
x = 0
sql_params = {
"version0": "WaterWolf",
"version1": "2.0"
}
sql_params_exp = {
"version0": "WaterWolf",
"version1": "2.0"
}
version_where_exp = (
"("
"r.product=%(version0)s"
" AND r.version=%(version1)s"
")"
)
version_where = pgbase.build_version_where(
"WaterWolf",
"2.0",
x,
sql_params,
params["versions_info"][key],
config,
)
eq_(version_where, version_where_exp)
eq_(sql_params, sql_params_exp)
def test_build_reports_sql_where(self):
""" Test PostgreSQLBase.build_reports_sql_where()."""
config = self.get_dummy_context()
pgbase = self.get_instance()
params = search_common.get_parameters({}) # Get default search params
default_params = util.DotDict(params.copy())
sql_params = {}
# .....................................................................
# Test 1: default values for parameters
sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \
"%(to_date)s"
sql_params_exp = {
"from_date": params.from_date,
"to_date": params.to_date
}
(sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params,
config)
sql = " ".join(sql.split()) # squeeze all \s, \r, \t...
eq_(sql, sql_exp)
eq_(sql_params, sql_params_exp)
# .....................................................................
# Test 2: terms and search_mode = is_exactly
sql_params = {}
params.terms = "signature"
params.search_mode = "is_exactly"
sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \
"%(to_date)s AND r.signature=%(term)s"
sql_params_exp = {
"from_date": params.from_date,
"to_date": params.to_date,
"term": params.terms
}
(sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params,
config)
sql = " ".join(sql.split()) # squeeze all \s, \r, \t...
eq_(sql, sql_exp)
eq_(sql_params, sql_params_exp)
# .....................................................................
# Test 3: terms and search_mode != is_exactly
sql_params = {}
params.terms = "signature%"
params.search_mode = "starts_with"
sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \
"%(to_date)s AND r.signature LIKE %(term)s"
sql_params_exp = {
"from_date": params.from_date,
"to_date": params.to_date,
"term": params.terms
}
(sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params,
config)
sql = " ".join(sql.split()) # squeeze all \s, \r, \t...
eq_(sql, sql_exp)
eq_(sql_params, sql_params_exp)
# .....................................................................
# Test 4: products
sql_params = {}
params.terms = default_params.terms
params.search_mode = default_params.search_mode
params.products = ["Firefox", "Fennec"]
sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \
"%(to_date)s AND (r.product=%(product0)s OR " \
"r.product=%(product1)s)"
sql_params_exp = {
"from_date": params.from_date,
"to_date": params.to_date,
"product0": "Firefox",
"product1": "Fennec"
}
(sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params,
config)
sql = " ".join(sql.split()) # squeeze all \s, \r, \t...
eq_(sql, sql_exp)
eq_(sql_params, sql_params_exp)
# .....................................................................
# Test 5: os
sql_params = {}
params.products = default_params.products
params.os = ["Windows"]
sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \
"%(to_date)s AND (r.os_name=%(os0)s)"
sql_params_exp = {
"from_date": params.from_date,
"to_date": params.to_date,
"os0": "Windows"
}
(sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params,
config)
sql = " ".join(sql.split()) # squeeze all \s, \r, \t...
eq_(sql, sql_exp)
eq_(sql_params, sql_params_exp)
# .....................................................................
# Test 6: build_ids
sql_params = {}
params.os = default_params.os
params.build_ids = ["20120101123456"]
sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \
"%(to_date)s AND (r.build=%(build0)s)"
sql_params_exp = {
"from_date": params.from_date,
"to_date": params.to_date,
"build0": "20120101123456"
}
(sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params,
config)
sql = " ".join(sql.split()) # squeeze all \s, \r, \t...
eq_(sql, sql_exp)
eq_(sql_params, sql_params_exp)
# .....................................................................
# Test 7: reasons
sql_params = {}
params.build_ids = default_params.build_ids
params.reasons = ["EXCEPTION", "OVERFLOW"]
sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \
"%(to_date)s AND (r.reason=%(reason0)s OR " \
"r.reason=%(reason1)s)"
sql_params_exp = {
"from_date": params.from_date,
"to_date": params.to_date,
"reason0": "EXCEPTION",
"reason1": "OVERFLOW"
}
(sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params,
config)
sql = " ".join(sql.split()) # squeeze all \s, \r, \t...
eq_(sql, sql_exp)
eq_(sql_params, sql_params_exp)
# .....................................................................
# Test 8: report_type
sql_params = {}
params.reasons = default_params.reasons
params.report_type = "crash"
sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \
"%(to_date)s AND r.hangid IS NULL"
sql_params_exp = {
"from_date": params.from_date,
"to_date": params.to_date
}
(sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params,
config)
sql = " ".join(sql.split()) # squeeze all \s, \r, \t...
eq_(sql, sql_exp)
eq_(sql_params, sql_params_exp)
# .....................................................................
# Test 9: versions
sql_params = {}
params.report_type = default_params.report_type
params.versions = ["Firefox", "12.0a1", "Fennec", "11.0", "Firefox",
"13.0b"]
params.versions_info = {
"Firefox:12.0a1": {
"version_string": "12.0a1",
"product_name": "Firefox",
"major_version": "12.0",
"release_channel": "Nightly",
"build_id": ["20120101123456"],
"is_rapid_beta": False,
"from_rapid_beta": False,
"from_beta_version": "Firefox:12.0a1",
},
"Fennec:11.0": {
"version_string": "11.0",
"product_name": "Fennec",
"major_version": None,
"release_channel": None,
"build_id": None,
"is_rapid_beta": False,
"from_rapid_beta": False,
"from_beta_version": "Fennec:11.0",
},
"Firefox:13.0b1": {
"version_string": "13.0b1",
"product_name": "Firefox",
"major_version": "13.0",
"release_channel": "Beta",
"build_id": ["20120101123456", "20120101098765"],
"is_rapid_beta": False,
"from_rapid_beta": True,
"from_beta_version": "Firefox:13.0b",
},
"Firefox:13.0b": {
"version_string": "13.0b",
"product_name": "Firefox",
"major_version": "13.0b",
"release_channel": "Beta",
"build_id": None,
"is_rapid_beta": True,
"from_rapid_beta": True,
"from_beta_version": "Firefox:13.0b",
}
}
sql_exp = """
WHERE r.date_processed BETWEEN %(from_date)s AND %(to_date)s
AND
((r.release_channel ILIKE 'nightly'
AND r.product=%(version0)s
AND r.version=%(version1)s)
OR (r.product=%(version2)s
AND r.version=%(version3)s)
OR (r.product=%(version4)s
AND r.version=%(version5)s)
OR (r.product=%(version6)s
AND r.version=%(version7)s)
OR (r.release_channel ILIKE 'beta'
AND r.build IN ('20120101123456', '20120101098765')
AND r.product=%(version8)s
AND r.version=%(version9)s)
OR (r.product=%(version10)s
AND r.version=%(version11)s))
"""
sql_params_exp = {
"from_date": params.from_date,
"to_date": params.to_date,
"version0": "Firefox",
"version1": "12.0a1",
"version2": "Firefox",
"version3": "12.0a1",
"version4": "Fennec",
"version5": "11.0",
"version6": "Fennec",
"version7": "11.0",
"version8": "Firefox",
"version9": "13.0b1",
"version10": "Firefox",
"version11": "13.0b",
}
(sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params,
config)
# squeeze all \s, \r, \t...
sql = " ".join(sql.split())
sql_exp = " ".join(sql_exp.split())
eq_(sql, sql_exp)
eq_(sql_params, sql_params_exp)
# .....................................................................
# Test 10: report_process = plugin
sql_params = {}
params.versions = default_params.versions
params.versions_infos = None
params.report_process = "plugin"
sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \
"%(to_date)s AND r.process_type = 'plugin' AND " \
"plugins_reports.date_processed BETWEEN " \
"%(from_date)s AND %(to_date)s"
sql_params_exp = {
"from_date": params.from_date,
"to_date": params.to_date
}
(sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params,
config)
sql = " ".join(sql.split()) # squeeze all \s, \r, \t...
eq_(sql, sql_exp)
eq_(sql_params, sql_params_exp)
# .....................................................................
# Test 11: report_process != plugin
sql_params = {}
params.report_process = "content"
sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \
"%(to_date)s AND r.process_type = 'content'"
sql_params_exp = {
"from_date": params.from_date,
"to_date": params.to_date
}
(sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params,
config)
sql = " ".join(sql.split()) # squeeze all \s, \r, \t...
eq_(sql, sql_exp)
eq_(sql_params, sql_params_exp)
# .....................................................................
# Test 12: plugins
sql_params = {}
params.report_process = "plugin"
params.plugin_terms = "plugin_name"
params.plugin_search_mode = "is_exactly"
params.plugin_in = ["name"]
sql_exp = "WHERE r.date_processed BETWEEN %(from_date)s AND " \
"%(to_date)s AND r.process_type = 'plugin' AND " \
"plugins_reports.date_processed BETWEEN " \
"%(from_date)s AND %(to_date)s AND " \
"(plugins.name=%(plugin_term)s)"
sql_params_exp = {
"from_date": params.from_date,
"to_date": params.to_date,
"plugin_term": params.plugin_terms
}
(sql, sql_params) = pgbase.build_reports_sql_where(params, sql_params,
config)
sql = " ".join(sql.split()) # squeeze all \s, \r, \t...
eq_(sql, sql_exp)
eq_(sql_params, sql_params_exp)
def setUp(self):
""" Populate product_info table with fake data """
super(IntegrationTestTCBS, self).setUp()
cursor = self.connection.cursor()
self.now = datetimeutil.utc_now()
now = self.now.date()
yesterday = now - datetime.timedelta(days=1)
lastweek = now - datetime.timedelta(days=7)
self.params = util.DotDict()
self.params.startDate = lastweek
self.params.product = 'Firefox'
self.params.version = '8.0'
self.params.limit = 100
self.params.to_date = yesterday
self.params.date_range_type = None
self.params.duration = datetime.timedelta(7)
self.params.os = None
self.params.crash_type = 'all'
self.params.logger = mock.Mock()
cursor.execute("""
INSERT INTO signatures
(signature_id, signature, first_report)
VALUES
(1, 'Fake Signature #1', '%(first_date)s'),
(2, 'Fake Signature #2', '%(first_date)s'),
(3, 'Fake Signature #3', '%(first_date)s');
""" % {'first_date': lastweek})
cursor.execute("""
INSERT INTO products
(product_name, sort, rapid_release_version, release_name)
VALUES
('Firefox', 1, '8.0', 'firefox'),
('Fennec', 2, '11.0', 'mobile');
""")
cursor.execute("""
INSERT INTO release_channels
(release_channel, sort)
VALUES
('Release', 1);
""")
cursor.execute("""
INSERT INTO product_versions
(product_version_id, product_name, major_version, release_version,
version_string, build_date, sunset_date, featured_version,
build_type)
VALUES
(
1,
'Firefox',
'8.0',
'8.0',
'8.0',
'%(start_date)s',
'%(end_date)s',
False,
'Release'
),
(
2,
'Fennec',
'11.0',
'11.0',
'11.0.1',
'%(start_date)s',
'%(end_date)s',
False,
'Release'
);
""" % {'start_date': self.now - datetime.timedelta(days=8),
'end_date': self.now})
cursor.execute("""
INSERT INTO signature_products_rollup
(signature_id, product_name, ver_count, version_list)
VALUES
(
1,
'Firefox',
2,
'{plugin1, plugin2}'
),
(
2,
'Firefox',
6,
'{plugin1, plugin2, plugin3, plugin4, plugin5, plugin6}'
),
(
3,
'Fennec',
3,
'{plugin1, plugin2}'
);
""")
cursor.execute("""
INSERT INTO tcbs
(
signature_id,
report_date,
product_version_id,
process_type,
release_channel,
report_count,
win_count,
mac_count,
lin_count,
hang_count,
is_gc_count
)
VALUES
(1, '%(lastweek)s', 1, 'type', 'Release', 14, 12, 1, 1, 0, 1),
(2, '%(now)s', 1, 'crash', 'Release', 3, 1, 1, 1, 0, 1),
(1, '%(now)s', 1, 'hang', 'Release', 5, 0, 0, 5, 5, 10),
(2, '%(lastweek)s', 1, 'crash', 'Release', 10, 7, 2, 1, 0, 3);
""" % {'now': self.now,
'lastweek': self.now - datetime.timedelta(days=8)})
self.connection.commit()
