def generate_ndt_query(self, start_time, end_time, metric, server_ips,
client_ip_blocks):
start_time_utc = utils.make_datetime_utc_aware(start_time)
end_time_utc = utils.make_datetime_utc_aware(end_time)
generator = query.BigQueryQueryGenerator(start_time_utc,
end_time_utc,
metric,
'ndt',
server_ips,
client_ip_blocks)
return generator.query()
def testValidDoubleSubset(self):
selector_file_contents = """{
"file_format_version": 1,
"duration": "30d",
"metric":"average_rtt",
"ip_translation":{
"strategy":"maxmind",
"params":{
"db_snapshots":["2014-08-04"]
}
},
"subsets":[
{
"site":"lga02",
"client_provider":"comcast",
"start_time":"2014-02-01T00:00:00Z"
},
{
"site":"lga01",
"client_provider":"comcast",
"start_time":"2014-02-01T00:00:00Z"
}
]
}"""
selector1_expected = selector.Selector()
selector1_expected.start_time = utils.make_datetime_utc_aware(
datetime.datetime(2014, 2, 1))
selector1_expected.duration = 30 * 24 * 60 * 60
selector1_expected.metric = 'average_rtt'
selector1_expected.site_name = 'lga02'
selector1_expected.ip_translation_spec = (
iptranslation.IPTranslationStrategySpec(
'maxmind', {'db_snapshots': ['2014-08-04']}))
selector1_expected.client_provider = 'comcast'
selector2_expected = selector.Selector()
selector2_expected.start_time = utils.make_datetime_utc_aware(
datetime.datetime(2014, 2, 1))
selector2_expected.duration = 30 * 24 * 60 * 60
selector2_expected.metric = 'average_rtt'
selector2_expected.ip_translation_spec = (
iptranslation.IPTranslationStrategySpec(
'maxmind', {'db_snapshots': ['2014-08-04']}))
selector2_expected.site_name = 'lga01'
selector2_expected.client_provider = 'comcast'
selectors_expected = [selector1_expected, selector2_expected]
self.assertParsedSelectorsMatch(selectors_expected,
selector_file_contents)
def testValidInput_v1dot1_EmptyListValue_OptionalParameter(self):
"""Empty list on optional parameter (client_countries) is handled as None"""
selector_file_contents = """{
"file_format_version": 1.1,
"duration": "30d",
"metrics": ["average_rtt"],
"ip_translation":{
"strategy":"maxmind",
"params":{
"db_snapshots":["2014-08-04"]
}
},
"client_countries": [],
"start_times": ["2014-02-01T00:00:00Z"]
}"""
selector_expected = selector.Selector()
selector_expected.start_time = utils.make_datetime_utc_aware(
datetime.datetime(2014, 2, 1))
selector_expected.duration = 30 * 24 * 60 * 60
selector_expected.metric = 'average_rtt'
selector_expected.ip_translation_spec = (
iptranslation.IPTranslationStrategySpec(
'maxmind', {'db_snapshots': ['2014-08-04']}))
selector_expected.client_country = None
self.assertParsedSingleSelectorMatches(selector_expected,
selector_file_contents)
def testValidInput_v1dot1_Simple_SingleLocationValues_onlyClientCountries(
self):
selector_file_contents = """{
"file_format_version": 1.1,
"duration": "30d",
"metrics": ["average_rtt"],
"ip_translation":{
"strategy":"maxmind",
"params":{
"db_snapshots":["2014-08-04"]
}
},
"client_countries": ["us"],
"start_times": ["2014-02-01T00:00:00Z"]
}"""
selector_expected = selector.Selector()
selector_expected.start_time = utils.make_datetime_utc_aware(
datetime.datetime(2014, 2, 1))
selector_expected.duration = 30 * 24 * 60 * 60
selector_expected.metric = 'average_rtt'
selector_expected.ip_translation_spec = (
iptranslation.IPTranslationStrategySpec(
'maxmind', {'db_snapshots': ['2014-08-04']}))
selector_expected.client_country = 'us'
self.assertParsedSingleSelectorMatches(selector_expected,
selector_file_contents)
def testValidInput_v1dot1_Simple_CountriesCaseInsensitivity(self):
selector_file_contents = """{
"file_format_version": 1.1,
"duration": "30d",
"metrics": ["average_rtt"],
"ip_translation":{
"strategy":"maxmind",
"params":{
"db_snapshots":["2014-08-04"]
}
},
"start_times": ["2014-02-01T00:00:00Z"],
"client_countries": ["us", "Ca", "uK", "AU"]
}"""
selectors_expected = []
selector_base = selector.Selector()
selector_base.start_time = utils.make_datetime_utc_aware(
datetime.datetime(2014, 2, 1))
selector_base.duration = 30 * 24 * 60 * 60
selector_base.metric = "average_rtt"
selector_base.ip_translation_spec = (
iptranslation.IPTranslationStrategySpec(
'maxmind', {'db_snapshots': ['2014-08-04']}))
for client_country in ('us', 'ca', 'uk', 'au'):
selector_copy = copy.copy(selector_base)
selector_copy.client_country = client_country
selectors_expected.append(selector_copy)
self.assertParsedSelectorsMatch(selectors_expected,
selector_file_contents)
def testSuccessfulParseOfValidv1_1FileWithAllOptionalFieldsDefined(self):
selector_file_contents = """{
"file_format_version": 1.1,
"duration": "30d",
"metrics": ["average_rtt"],
"ip_translation":{
"strategy":"maxmind",
"params":{
"db_snapshots":["2014-08-04"]
}
},
"sites": ["lga02"],
"client_providers": ["comcast"],
"client_countries": ["us"],
"start_times": ["2014-02-01T00:00:00Z"]
}"""
selector_expected = selector.Selector()
selector_expected.start_time = utils.make_datetime_utc_aware(
datetime.datetime(2014, 2, 1))
selector_expected.duration = 30 * 24 * 60 * 60
selector_expected.metric = 'average_rtt'
selector_expected.ip_translation_spec = (
iptranslation.IPTranslationStrategySpec(
'maxmind', {'db_snapshots': ['2014-08-04']}))
selector_expected.site = 'lga02'
selector_expected.client_provider = 'comcast'
selector_expected.client_country = 'us'
self.assertParsedSingleSelectorMatches(selector_expected,
selector_file_contents)
def testValidSingleSubset(self):
selector_file_contents = """{
"file_format_version": 1,
"duration": "30d",
"metric":"average_rtt",
"ip_translation":{
"strategy":"maxmind",
"params":{
"db_snapshots":["2014-08-04"]
}
},
"subsets":[
{
"site":"lga02",
"client_provider":"comcast",
"start_time":"2014-02-01T00:00:00Z"
}
]
}"""
selector_expected = selector.Selector()
selector_expected.start_time = utils.make_datetime_utc_aware(datetime.datetime(2014, 2, 1))
selector_expected.duration = 30 * 24 * 60 * 60
selector_expected.metric = 'average_rtt'
selector_expected.ip_translation_spec = (
iptranslation.IPTranslationStrategySpec('maxmind',
{'db_snapshots': ['2014-08-04']}))
selector_expected.site_name = 'lga02'
selector_expected.client_provider = 'comcast'
self.assertParsedSingleSelectorMatches(selector_expected, selector_file_contents)
def testValidInput_v1dot1_Simple_CountriesCaseInsensitivity(self):
selector_file_contents = """{
"file_format_version": 1.1,
"duration": "30d",
"metrics": ["average_rtt"],
"ip_translation":{
"strategy":"maxmind",
"params":{
"db_snapshots":["2014-08-04"]
}
},
"start_times": ["2014-02-01T00:00:00Z"],
"client_countries": ["us", "Ca", "uK", "AU"]
}"""
selectors_expected = []
selector_base = selector.Selector()
selector_base.start_time = utils.make_datetime_utc_aware(
datetime.datetime(2014, 2, 1))
selector_base.duration = 30 * 24 * 60 * 60
selector_base.metric = "average_rtt"
selector_base.ip_translation_spec = (
iptranslation.IPTranslationStrategySpec(
'maxmind', {'db_snapshots': ['2014-08-04']}))
for client_country in ('us', 'ca', 'uk', 'au'):
selector_copy = copy.copy(selector_base)
selector_copy.client_country = client_country
selectors_expected.append(selector_copy)
self.assertParsedSelectorsMatch(selectors_expected,
selector_file_contents)
def testValidInput_v1dot1_EmptyListValue_OptionalParameter(self):
"""Empty list on optional parameter (client_countries) is handled as None"""
selector_file_contents = """{
"file_format_version": 1.1,
"duration": "30d",
"metrics": ["average_rtt"],
"ip_translation":{
"strategy":"maxmind",
"params":{
"db_snapshots":["2014-08-04"]
}
},
"client_countries": [],
"start_times": ["2014-02-01T00:00:00Z"]
}"""
selector_expected = selector.Selector()
selector_expected.start_time = utils.make_datetime_utc_aware(
datetime.datetime(2014, 2, 1))
selector_expected.duration = 30 * 24 * 60 * 60
selector_expected.metric = 'average_rtt'
selector_expected.ip_translation_spec = (
iptranslation.IPTranslationStrategySpec(
'maxmind', {'db_snapshots': ['2014-08-04']}))
selector_expected.client_country = None
self.assertParsedSingleSelectorMatches(selector_expected,
selector_file_contents)
def testValidInput_v1dot1_Simple_SingleLocationValues_onlyClientCountries(
self):
selector_file_contents = """{
"file_format_version": 1.1,
"duration": "30d",
"metrics": ["average_rtt"],
"ip_translation":{
"strategy":"maxmind",
"params":{
"db_snapshots":["2014-08-04"]
}
},
"client_countries": ["us"],
"start_times": ["2014-02-01T00:00:00Z"]
}"""
selector_expected = selector.Selector()
selector_expected.start_time = utils.make_datetime_utc_aware(
datetime.datetime(2014, 2, 1))
selector_expected.duration = 30 * 24 * 60 * 60
selector_expected.metric = 'average_rtt'
selector_expected.ip_translation_spec = (
iptranslation.IPTranslationStrategySpec(
'maxmind', {'db_snapshots': ['2014-08-04']}))
selector_expected.client_country = 'us'
self.assertParsedSingleSelectorMatches(selector_expected,
selector_file_contents)
def testSuccessfulParseOfValidv1_1FileWithAllOptionalFieldsDefined(self):
selector_file_contents = """{
"file_format_version": 1.1,
"duration": "30d",
"metrics": ["average_rtt"],
"ip_translation":{
"strategy":"maxmind",
"params":{
"db_snapshots":["2014-08-04"]
}
},
"sites": ["lga02"],
"client_providers": ["comcast"],
"client_countries": ["us"],
"start_times": ["2014-02-01T00:00:00Z"]
}"""
selector_expected = selector.Selector()
selector_expected.start_time = utils.make_datetime_utc_aware(
datetime.datetime(2014, 2, 1))
selector_expected.duration = 30 * 24 * 60 * 60
selector_expected.metric = 'average_rtt'
selector_expected.ip_translation_spec = (
iptranslation.IPTranslationStrategySpec(
'maxmind', {'db_snapshots': ['2014-08-04']}))
selector_expected.site = 'lga02'
selector_expected.client_provider = 'comcast'
selector_expected.client_country = 'us'
self.assertParsedSingleSelectorMatches(selector_expected,
selector_file_contents)
def testValidSingleSubsetAllMetric(self):
selector_file_contents = """{
"file_format_version": 1,
"duration": "30d",
"metric":"all",
"ip_translation":{
"strategy":"maxmind",
"params":{
"db_snapshots":["2014-08-04"]
}
},
"subsets":[
{
"site":"lga02",
"client_provider":"comcast",
"start_time":"2014-02-01T00:00:00Z"
}
]
}"""
selector_base = selector.Selector()
selector_base.start_time = utils.make_datetime_utc_aware(datetime.datetime(2014, 2, 1))
selector_base.duration = 30 * 24 * 60 * 60
selector_base.metric = 'average_rtt'
selector_base.ip_translation_spec = (
iptranslation.IPTranslationStrategySpec('maxmind',
{'db_snapshots': ['2014-08-04']}))
selector_base.site_name = 'lga02'
selector_base.client_provider = 'comcast'
selectors_expected = []
# TODO(mtlynch): Need to fix this test so that changing the order of the
# expected_metrics list doesn't cause the test to fail.
expected_metrics = (
'minimum_rtt',
'upload_throughput',
'packet_retransmit_rate',
'download_throughput',
'average_rtt'
)
for metric in expected_metrics:
selector_copy = copy.copy(selector_base)
selector_copy.metric = metric
selectors_expected.append(selector_copy)
# The 'all' metric should expand to selectors for every supported metric.
self.assertParsedSelectorsMatch(selectors_expected, selector_file_contents)
def testValidSingleSubsetAllMetric(self):
selector_file_contents = """{
"file_format_version": 1,
"duration": "30d",
"metric":"all",
"ip_translation":{
"strategy":"maxmind",
"params":{
"db_snapshots":["2014-08-04"]
}
},
"subsets":[
{
"site":"lga02",
"client_provider":"comcast",
"start_time":"2014-02-01T00:00:00Z"
}
]
}"""
selector_base = selector.Selector()
selector_base.start_time = utils.make_datetime_utc_aware(
datetime.datetime(2014, 2, 1))
selector_base.duration = 30 * 24 * 60 * 60
selector_base.metric = 'average_rtt'
selector_base.ip_translation_spec = (
iptranslation.IPTranslationStrategySpec(
'maxmind', {'db_snapshots': ['2014-08-04']}))
selector_base.site_name = 'lga02'
selector_base.client_provider = 'comcast'
selectors_expected = []
# TODO(mtlynch): Need to fix this test so that changing the order of the
# expected_metrics list doesn't cause the test to fail.
expected_metrics = ('minimum_rtt', 'upload_throughput',
'packet_retransmit_rate', 'download_throughput',
'average_rtt')
for metric in expected_metrics:
selector_copy = copy.copy(selector_base)
selector_copy.metric = metric
selectors_expected.append(selector_copy)
# The 'all' metric should expand to selectors for every supported metric.
self.assertParsedSelectorsMatch(selectors_expected,
selector_file_contents)
def _parse_start_time(self, start_time_string):
"""Parse the time window start time.
Parse the start time from the expected timestamp format to Python datetime
format. Must be in UTC time.
Args:
start_time_string: (str) Timestamp in format YYYY-MM-DDTHH-mm-SS.
Returns:
datetime: Python datetime for set timestamp string.
"""
try:
timestamp = (
datetime.datetime.strptime(start_time_string, '%Y-%m-%dT%H:%M:%SZ'))
return utils.make_datetime_utc_aware(timestamp)
except ValueError:
raise ValueError('UnsupportedSubsetDateFormat')
def _parse_start_time(self, start_time_string):
"""Parse the time window start time.
Parse the start time from the expected timestamp format to Python datetime
format. Must be in UTC time.
Args:
start_time_string: (str) Timestamp in format YYYY-MM-DDTHH-mm-SS.
Returns:
datetime: Python datetime for set timestamp string.
"""
try:
timestamp = (datetime.datetime.strptime(start_time_string,
'%Y-%m-%dT%H:%M:%SZ'))
return utils.make_datetime_utc_aware(timestamp)
except ValueError:
raise ValueError('UnsupportedSubsetDateFormat')
def testValidInput_v1dot1_Complex(self):
selector_file_contents = """{
"file_format_version": 1.1,
"duration": "30d",
"metrics": ["minimum_rtt", "download_throughput", "average_rtt"],
"ip_translation":{
"strategy":"maxmind",
"params":{
"db_snapshots":["2014-08-04"]
}
},
"sites": ["lga01", "lga02"],
"client_providers": ["comcast", "verizon"],
"start_times": ["2014-02-01T00:00:00Z"]
}"""
selectors_expected = []
selector_base = selector.Selector()
selector_base.start_time = utils.make_datetime_utc_aware(
datetime.datetime(2014, 2, 1))
selector_base.duration = 30 * 24 * 60 * 60
selector_base.ip_translation_spec = (
iptranslation.IPTranslationStrategySpec(
'maxmind', {'db_snapshots': ['2014-08-04']}))
sites = ['lga01', 'lga02']
client_providers = ['comcast', 'verizon']
metrics = ['minimum_rtt', 'download_throughput', 'average_rtt']
for client_provider, site, metric in itertools.product(
client_providers, sites, metrics):
selector_copy = copy.copy(selector_base)
selector_copy.metric = metric
selector_copy.client_provider = client_provider
selector_copy.site = site
selectors_expected.append(selector_copy)
self.assertParsedSelectorsMatch(selectors_expected,
selector_file_contents)
def testValidInput_v1dot1_Complex(self):
selector_file_contents = """{
"file_format_version": 1.1,
"duration": "30d",
"metrics": ["minimum_rtt", "download_throughput", "average_rtt"],
"ip_translation":{
"strategy":"maxmind",
"params":{
"db_snapshots":["2014-08-04"]
}
},
"sites": ["lga01", "lga02"],
"client_providers": ["comcast", "verizon"],
"start_times": ["2014-02-01T00:00:00Z"]
}"""
selectors_expected = []
selector_base = selector.Selector()
selector_base.start_time = utils.make_datetime_utc_aware(
datetime.datetime(2014, 2, 1))
selector_base.duration = 30 * 24 * 60 * 60
selector_base.ip_translation_spec = (
iptranslation.IPTranslationStrategySpec(
'maxmind', {'db_snapshots': ['2014-08-04']}))
sites = ['lga01', 'lga02']
client_providers = ['comcast', 'verizon']
metrics = ['minimum_rtt', 'download_throughput', 'average_rtt']
for client_provider, site, metric in itertools.product(
client_providers, sites, metrics):
selector_copy = copy.copy(selector_base)
selector_copy.metric = metric
selector_copy.client_provider = client_provider
selector_copy.site = site
selectors_expected.append(selector_copy)
self.assertParsedSelectorsMatch(selectors_expected,
selector_file_contents)
