def test_init_ds_args(self):
"""Make sure that just passing in data source information constructs
an AggregatingDataSource with the expected attributes.
"""
expected_data_sources = [{
'data_server_url':
"http://a.com",
'data_source_hash':
util.generate_ds_key("some.data.source"),
'secret_key':
"TEST_SECRET_ONE"
}, {
'data_server_url':
"http://b.com",
'data_source_hash':
util.generate_ds_key("another.data.source"),
'secret_key':
"TEST_SECRET_TWO"
}, {
'data_server_url': "http://c.com",
'data_source_hash': 'ae22d4c',
'secret_key': "TEST_SECRET_THREE"
}]
T.assert_equal(self.data_source.data_sources, expected_data_sources)
desc_data_source = AggregatingDataSource(
data_sources=self.test_data_sources, desc=mock.sentinel.test_desc)
T.assert_equal(desc_data_source.data_sources, expected_data_sources)
T.assert_equal(desc_data_source.DESC, mock.sentinel.test_desc)
def setup_data_source(self): self.data_source = AggregatingDataSource(data_sources=self.test_data_sources)
class AggregatingDataSourceTest(T.TestCase):
test_data_sources = [
{
'data_server_url': "http://a.com",
'data_source_name': "some.data.source",
'secret_key': "TEST_SECRET_ONE"
},
{
'data_server_url': "http://b.com",
'data_source_name': "another.data.source",
'name_is_hash': False,
'secret_key': "TEST_SECRET_TWO"
},
{
'data_server_url': "http://c.com",
'data_source_name': 'ae22d4c',
'name_is_hash': True,
'secret_key': "TEST_SECRET_THREE"
}
]
@T.setup
def setup_data_source(self):
self.data_source = AggregatingDataSource(data_sources=self.test_data_sources)
def test_init_ds_args(self):
"""Make sure that just passing in data source information constructs
an AggregatingDataSource with the expected attributes.
"""
expected_data_sources = [
{
'data_server_url': "http://a.com",
'data_source_hash': util.generate_ds_key("some.data.source"),
'secret_key': "TEST_SECRET_ONE"
},
{
'data_server_url': "http://b.com",
'data_source_hash': util.generate_ds_key("another.data.source"),
'secret_key': "TEST_SECRET_TWO"
},
{
'data_server_url': "http://c.com",
'data_source_hash': 'ae22d4c',
'secret_key': "TEST_SECRET_THREE"
}
]
T.assert_equal(self.data_source.data_sources, expected_data_sources)
desc_data_source = AggregatingDataSource(data_sources=self.test_data_sources,
desc=mock.sentinel.test_desc)
T.assert_equal(desc_data_source.data_sources, expected_data_sources)
T.assert_equal(desc_data_source.DESC, mock.sentinel.test_desc)
def test_list_path_no_path(self):
"""Tests the behavior of list_path when asking for the root keys (no
path specified).
"""
test_path = None
expected_paths = [{
'name': 'src.MajorSource%d' % i,
'type': 'dir'
} for i in xrange(len(self.data_source.data_sources))]
with self._mock_ds_method('_request_paths_from_ds') as mock_request_paths:
mock_request_paths.side_effect = [[path] for path in expected_paths]
actual_paths = self.data_source.list_path(test_path)
T.assert_equal(mock_request_paths.call_count, len(self.data_source.data_sources))
for ds in self.data_source.data_sources:
mock_request_paths.assert_any_call(ds, test_path)
T.assert_equal(expected_paths, actual_paths)
for expected_path, expected_data_source in zip(expected_paths, self.data_source.data_sources):
T.assert_in(expected_path['name'], self.data_source.key_mapping_cache)
T.assert_equal(self.data_source.key_mapping_cache[expected_path['name']], expected_data_source)
def test_list_path_no_path_duplicates(self):
"""Tests that when no path is specified the correct results are returned
and the repeat key is not cached with the wrong data source.
"""
test_path = None
expected_paths = [{
'name': 'src.MajorSource%d' % i,
'type': 'dir'
} for i in xrange(len(self.data_source.data_sources))]
with self._mock_ds_method('_request_paths_from_ds') as mock_request_paths:
mock_request_path_list = [[path] for path in expected_paths]
mock_request_path_list[-1].append({
'name': 'src.MajorSource1',
'type': 'dir'
})
mock_request_paths.side_effect = mock_request_path_list
actual_paths = self.data_source.list_path(test_path)
T.assert_equal(mock_request_paths.call_count, len(self.data_source.data_sources))
for ds in self.data_source.data_sources:
mock_request_paths.assert_any_call(ds, test_path)
T.assert_equal(expected_paths, actual_paths)
for expected_path, expected_data_source in zip(expected_paths, self.data_source.data_sources):
T.assert_in(expected_path['name'], self.data_source.key_mapping_cache)
T.assert_equal(self.data_source.key_mapping_cache[expected_path['name']], expected_data_source)
def test_list_path(self):
"""Tests the behavior of list_path when a path is specified."""
test_path = ['src.MajorSource1']
expected_paths = [{
'name': 'src.minorSource%d' % i,
'type': 'dir'
} for i in xrange(5)]
with self._mock_ds_method('_request_paths_from_ds') as mock_request_paths:
with self._mock_ds_method('_data_source_for_stat_key') as mock_ds_for_stat_key:
mock_ds_for_stat_key.return_value = self.data_source.data_sources[1]
mock_request_paths.return_value = [path for path in expected_paths]
actual_paths = self.data_source.list_path(test_path)
mock_ds_for_stat_key.assert_called_once_with(test_path[0])
mock_request_paths.assert_called_once_with(self.data_source.data_sources[1], test_path)
T.assert_equal(expected_paths, actual_paths)
def test_data_single_from_single_source(self):
"""Tests the behavior of data when asking for a single source."""
test_data_source = self.data_source.data_sources[2]
test_stat = [['src.MajorSourceA', 'key.minorStat1', 'variant.Blue', '50th_percentile']]
test_start = 100
test_end = 200
test_width = 500
dummy_data = [
{
't': 500,
'v': [120]
},
{
't': 505,
'v': [140]
},
{
't': 510,
'v': [160]
}
]
expected_data = json.dumps(dummy_data)
with self._mock_ds_method('_data_source_for_stat_key') as mock_ds_for_stat_key:
with self._mock_ds_method('_request_data_from_ds') as mock_request_data:
mock_ds_for_stat_key.return_value = test_data_source
mock_request_data.return_value = dummy_data
actual_data = self.data_source.data(test_stat, test_start, test_end, test_width)
mock_ds_for_stat_key.assert_called_once_with(test_stat[0][0])
mock_request_data.assert_called_once_with(test_data_source, test_stat, test_start, test_end, test_width)
T.assert_equal(expected_data, actual_data)
def test_data_multiple_from_single_source(self):
"""Tests the behavior of data when asking for multiple stats from a
single source.
"""
test_data_source = self.data_source.data_sources[2]
test_stats = [
['src.MajorSourceA', 'key.minorStat1', 'variant.Blue', '50th_percentile'],
['src.MajorSourceA', 'key.minorStat1', 'variant.Red', '75th_percentile']
]
test_start = 100
test_end = 200
test_width = 500
dummy_data = [
{
't': 500,
'v': [120, 190]
},
{
't': 505,
'v': [140, 200]
},
{
't': 510,
'v': [160, 40]
}
]
expected_data = json.dumps(dummy_data)
with self._mock_ds_method('_data_source_for_stat_key') as mock_ds_for_stat_key:
with self._mock_ds_method('_request_data_from_ds') as mock_request_data:
mock_ds_for_stat_key.side_effect = [test_data_source, test_data_source]
mock_request_data.return_value = dummy_data
actual_data = self.data_source.data(test_stats, test_start, test_end, test_width)
T.assert_equal(mock_ds_for_stat_key.call_count, len(test_stats))
for stat in test_stats:
mock_ds_for_stat_key.assert_any_call(stat[0])
mock_request_data.assert_called_once_with(test_data_source, test_stats, test_start, test_end, test_width)
T.assert_equal(expected_data, actual_data)
def test_data_multiple_from_multiple_sources(self):
"""Tests the behavior of data when asking for multiple stats from
multiple sources.
"""
# A -> 0, B -> 1, C -> 2 in self.data_source.data_sources
test_stats = [
['src.MajorSourceA', 'key.minorStat1', 'variant.Blue', '50th_percentile'],
['src.MajorSourceB', 'key.minorStatGolf', 'variant.eskimo', 'count'],
['src.MajorSourceA', 'key.minorStat1', 'variant.Red', '75th_percentile'],
['src.MajorSourceC', 'key.minorStatPidgeon', 'variant.alice', 'dart']
]
test_start = 100
test_end = 200
test_width = 500
# Lists data in the order of the data sources that will be asked
dummy_data_list = [
[
{
't': 500,
'v': [120, 190]
},
{
't': 505,
'v': [140, 200]
},
{
't': 510,
'v': [160, 40]
}
],
[
{
't': 496,
'v': [90]
},
{
't': 499,
'v': [93]
},
{
't': 505,
'v': [101]
},
{
't': 510,
'v': [100]
}
],
[
{
't': 500,
'v': [4]
},
{
't': 507,
'v': [10]
},
{
't': 510,
'v': [4]
},
{
't': 512,
'v': [8]
}
]
]
# Derived from dummy_data_list
expected_data = json.dumps([
{
't': 496,
'v': [None, 90, None, None]
},
{
't': 499,
'v': [None, 93, None, None]
},
{
't': 500,
'v': [120, None, 190, 4]
},
{
't': 505,
'v': [140, 101, 200, None]
},
{
't': 507,
'v': [None, None, None, 10]
},
{
't': 510,
'v': [160, 100, 40, 4]
},
{
't': 512,
'v': [None, None, None, 8]
}
])
with self._mock_ds_method('_data_source_for_stat_key') as mock_ds_for_stat_key:
with self._mock_ds_method('_request_data_from_ds') as mock_request_data:
mock_ds_for_stat_key.side_effect = [
self.data_source.data_sources[0],
self.data_source.data_sources[1],
self.data_source.data_sources[0],
self.data_source.data_sources[2],
]
mock_request_data.side_effect = dummy_data_list
actual_data = self.data_source.data(test_stats, test_start, test_end, test_width)
T.assert_equal(mock_ds_for_stat_key.call_count, len(test_stats))
for stat in test_stats:
mock_ds_for_stat_key.assert_any_call(stat[0])
T.assert_equal(mock_request_data.call_count, 3)
mock_request_data.assert_any_call(self.data_source.data_sources[0],
[test_stats[0], test_stats[2]], test_start, test_end, test_width)
mock_request_data.assert_any_call(self.data_source.data_sources[1],
[test_stats[1]], test_start, test_end, test_width)
mock_request_data.assert_any_call(self.data_source.data_sources[2],
[test_stats[3]], test_start, test_end, test_width)
T.assert_equal(expected_data, actual_data)
def test_int_dict_to_list(self):
"""Just a quick test for the positive path of _int_dict_to_list."""
test_dict = {
1: 'a',
2: 'b',
3: ['c', {'a': 'i'}],
4: 0,
11: 'cya'
}
expected_list = ['a', 'b', ['c', {'a': 'i'}], 0, 'cya']
actual_list = self.data_source._int_dict_to_list(test_dict)
T.assert_equal(expected_list, actual_list)
def test_request_paths_from_ds(self):
"""Tests the behavior of _request_paths_from_ds when a path is
specified.
"""
test_data_source = self.data_source.data_sources[0]
test_path = ['src.A']
mock_paths = ['key.D', 'key.E', 'key.F']
expected_ask_path = json.dumps([test_data_source['data_source_hash']] + test_path)
expected_paths = mock_paths
expected_token = util.generate_access_token(test_data_source['secret_key'])
with self._patch_urlopen() as mock_urlopen:
mock_urlopen.return_value = StringIO(json.dumps(mock_paths))
actual_paths = self.data_source._request_paths_from_ds(test_data_source, test_path)
T.assert_equal(mock_urlopen.call_count, 1)
T.assert_equal(mock_paths, actual_paths) # 'cause why not?
T.assert_equal(len(mock_urlopen.call_args[0]), 1)
submitted_url = mock_urlopen.call_args[0][0]
parsed_url = urlparse.urlparse(submitted_url)
query_params = urlparse.parse_qs(parsed_url.query)
T.assert_equal(parsed_url.scheme, 'http')
T.assert_equal(parsed_url.path, '/sources')
T.assert_equal(parsed_url.fragment, '')
T.assert_equal(query_params['path'], [expected_ask_path])
T.assert_equal(query_params['token'], [expected_token])
T.assert_equal(actual_paths, expected_paths)
def test_request_paths_from_ds_no_path(self):
"""Tests the behavior of _request_paths_from_ds when no path is
specified."""
test_data_source = self.data_source.data_sources[0]
test_path = None
mock_paths = ['src.A', 'src.B', 'src.C']
expected_ask_path = json.dumps([test_data_source['data_source_hash']])
expected_paths = mock_paths
expected_token = util.generate_access_token(test_data_source['secret_key'])
with self._patch_urlopen() as mock_urlopen:
mock_urlopen.return_value = StringIO(json.dumps(mock_paths))
actual_paths = self.data_source._request_paths_from_ds(test_data_source, test_path)
T.assert_equal(mock_urlopen.call_count, 1)
T.assert_equal(mock_paths, actual_paths) # 'cause why not?
T.assert_equal(len(mock_urlopen.call_args[0]), 1)
submitted_url = mock_urlopen.call_args[0][0]
parsed_url = urlparse.urlparse(submitted_url)
query_params = urlparse.parse_qs(parsed_url.query)
T.assert_equal(parsed_url.scheme, 'http')
T.assert_equal(parsed_url.path, '/sources')
T.assert_equal(parsed_url.fragment, '')
T.assert_equal(query_params['path'], [expected_ask_path])
T.assert_equal(query_params['token'], [expected_token])
T.assert_equal(actual_paths, expected_paths)
def test_request_paths_from_ds_url_error(self):
"""Tests that _request_paths_from_ds returns an empty list if it
can't contact the data_server.
"""
test_data_source = self.data_source.data_sources[0]
test_path = None
expected_paths = []
with self._patch_urlopen() as mock_urlopen:
with self._mock_ds_method('logger') as mock_logger:
# Set urlopen raise a URLError
mock_urlopen.side_effect = urllib2.URLError("Couldn't talk to the data server")
actual_paths = self.data_source._request_paths_from_ds(test_data_source, test_path)
T.assert_equal(mock_urlopen.call_count, 1)
T.assert_equal(actual_paths, expected_paths)
mock_logger.exception.assert_called_once_with("Failed to fetch paths for %s from %s" % (test_path, test_data_source['data_server_url']))
def test_request_paths_from_ds_invalid_response(self):
"""Tests that _request_paths_from_ds returns an empty list if it
can't parse the result of its interaction with the data server.
"""
test_data_source = self.data_source.data_sources[0]
test_path = None
expected_paths = []
with self._patch_urlopen() as mock_urlopen:
with self._mock_ds_method('logger') as mock_logger:
# Set urlopen to return a result that can't be JSON parsed
mock_urlopen.return_value = StringIO("[{{1_\]")
actual_paths = self.data_source._request_paths_from_ds(test_data_source, test_path)
T.assert_equal(mock_urlopen.call_count, 1)
T.assert_equal(actual_paths, expected_paths)
mock_logger.exception.assert_called_once_with("Invalid response received from %s" % test_data_source['data_server_url'])
def test_request_data_from_ds(self):
"""Checks that _request_data_from_ds knows how to ask for data from
other data servers correctly.
TODO(fhats): This can certainly benefit from some integration tests.
"""
test_data_source = self.data_source.data_sources[0]
test_sources = [
['src.EndpointTIming', 'stat.A', 'variant.logged_in'],
['src.ErrorCount', 'stat.B', 'variant.logged_out']
]
test_start = 100
test_end = 200
test_width = 30
expected_sources = map(lambda x: [test_data_source['data_source_hash']] + x, test_sources)
expected_token = util.generate_access_token(test_data_source['secret_key'])
with self._patch_urlopen() as mock_urlopen:
# Just sprinkle some mock data to return
mock_data = [
{
't': 150,
'v': [5, 6]
},
{
't': 160,
'v': [9, 10]
}
]
mock_urlopen.return_value = StringIO(json.dumps(mock_data))
returned_data = self.data_source._request_data_from_ds(test_data_source,
test_sources,
test_start,
test_end,
test_width)
T.assert_equal(mock_urlopen.call_count, 1)
T.assert_equal(mock_data, returned_data) # 'cause why not?
T.assert_equal(len(mock_urlopen.call_args[0]), 1)
submitted_url = mock_urlopen.call_args[0][0]
parsed_url = urlparse.urlparse(submitted_url)
query_params = urlparse.parse_qs(parsed_url.query)
T.assert_equal(parsed_url.scheme, 'http')
T.assert_equal(parsed_url.path, '/data')
T.assert_equal(parsed_url.fragment, '')
T.assert_equal(query_params['sources'], [json.dumps(expected_sources)])
T.assert_equal(query_params['start'], [str(test_start)])
T.assert_equal(query_params['end'], [str(test_end)])
T.assert_equal(query_params['width'], [str(test_width)])
T.assert_equal(query_params['token'], [expected_token])
def test_data_source_for_stat_key_in_cache(self):
"""Tests that _data_source_for_stat_key uses the key_mapping_cache if
it can.
"""
expected_data_source = self.data_source.data_sources[2]
stat_key = 'src.test_stat_key'
self.data_source.key_mapping_cache[stat_key] = expected_data_source
with self._mock_ds_method('_find_data_source_for_stat_key') as mock_find_ds_for_stat:
actual_ds = self.data_source._data_source_for_stat_key(stat_key)
T.assert_equal(mock_find_ds_for_stat.call_count, 0)
T.assert_equal(actual_ds, expected_data_source)
def test_data_source_for_stat_key_not_cached(self):
"""Tests that _data_source_for_stat_key asks _find_data_source_for_stat_key
if the stat key isn't cached in the key_mapping_cache.
"""
expected_data_source = self.data_source.data_sources[2]
stat_key = 'src.test_stat_key'
with self._mock_ds_method('_find_data_source_for_stat_key') as mock_find_ds_for_stat:
mock_find_ds_for_stat.return_value = expected_data_source
actual_ds = self.data_source._data_source_for_stat_key(stat_key)
mock_find_ds_for_stat.assert_called_once_with(stat_key)
T.assert_equal(actual_ds, expected_data_source)
def test_find_data_source_for_stat_key(self):
"""Tests _find_data_source_for_stat_key when it's provided by one of
the configured data sources.
"""
expected_data_source = {
'data_server_url': "http://b.com",
'data_source_hash': util.generate_ds_key("another.data.source"),
'secret_key': "TEST_SECRET_TWO"
}
test_key = 'src.our_key'
def fake_paths_from_ds(data_source, path):
if data_source == expected_data_source:
return [{"name": test_key},]
else:
return [{"name": "src.not_our_key"},]
with mock.patch.object(self.data_source, '_request_paths_from_ds', fake_paths_from_ds):
actual_ds = self.data_source._find_data_source_for_stat_key(test_key)
T.assert_equal(expected_data_source, actual_ds)
T.assert_in(test_key, self.data_source.key_mapping_cache)
T.assert_equal(expected_data_source, self.data_source.key_mapping_cache[test_key])
def test_find_data_source_for_stat_key_not_found(self):
"""Tests _find_data_source_for_stat_key's behavior when the stat_key
isn't made available by any data sources.
"""
expected_data_source = None
test_key = 'src.some_test_stat'
with self._mock_ds_method('_request_paths_from_ds') as mock_request_paths:
mock_request_paths.return_value = [{"name": "a"}, {"name": "b"}, {"name": "c"}]
actual_ds = self.data_source._find_data_source_for_stat_key(test_key)
T.assert_equal(mock_request_paths.call_count, len(self.test_data_sources))
for ds in self.data_source.data_sources:
mock_request_paths.assert_any_call(ds, None)
T.assert_equal(expected_data_source, actual_ds)
@contextmanager
def _patch_urlopen(self):
with mock.patch("firefly.data_sources.aggregating_data_source.urlopen") as mock_urlopen:
yield mock_urlopen
@contextmanager
def _mock_ds_method(self, method):
with mock.patch.object(self.data_source, method) as mock_thing:
yield mock_thing
def setup_data_source(self):
self.data_source = AggregatingDataSource(
data_sources=self.test_data_sources)
class AggregatingDataSourceTest(T.TestCase):
test_data_sources = [{
'data_server_url': "http://a.com",
'data_source_name': "some.data.source",
'secret_key': "TEST_SECRET_ONE"
}, {
'data_server_url': "http://b.com",
'data_source_name': "another.data.source",
'name_is_hash': False,
'secret_key': "TEST_SECRET_TWO"
}, {
'data_server_url': "http://c.com",
'data_source_name': 'ae22d4c',
'name_is_hash': True,
'secret_key': "TEST_SECRET_THREE"
}]
@T.setup
def setup_data_source(self):
self.data_source = AggregatingDataSource(
data_sources=self.test_data_sources)
def test_init_ds_args(self):
"""Make sure that just passing in data source information constructs
an AggregatingDataSource with the expected attributes.
"""
expected_data_sources = [{
'data_server_url':
"http://a.com",
'data_source_hash':
util.generate_ds_key("some.data.source"),
'secret_key':
"TEST_SECRET_ONE"
}, {
'data_server_url':
"http://b.com",
'data_source_hash':
util.generate_ds_key("another.data.source"),
'secret_key':
"TEST_SECRET_TWO"
}, {
'data_server_url': "http://c.com",
'data_source_hash': 'ae22d4c',
'secret_key': "TEST_SECRET_THREE"
}]
T.assert_equal(self.data_source.data_sources, expected_data_sources)
desc_data_source = AggregatingDataSource(
data_sources=self.test_data_sources, desc=mock.sentinel.test_desc)
T.assert_equal(desc_data_source.data_sources, expected_data_sources)
T.assert_equal(desc_data_source.DESC, mock.sentinel.test_desc)
def test_list_path_no_path(self):
"""Tests the behavior of list_path when asking for the root keys (no
path specified).
"""
test_path = None
expected_paths = [{
'name': 'src.MajorSource%d' % i,
'type': 'dir'
} for i in xrange(len(self.data_source.data_sources))]
with self._mock_ds_method(
'_request_paths_from_ds') as mock_request_paths:
mock_request_paths.side_effect = [[path]
for path in expected_paths]
actual_paths = self.data_source.list_path(test_path)
T.assert_equal(mock_request_paths.call_count,
len(self.data_source.data_sources))
for ds in self.data_source.data_sources:
mock_request_paths.assert_any_call(ds, test_path)
T.assert_equal(expected_paths, actual_paths)
for expected_path, expected_data_source in zip(
expected_paths, self.data_source.data_sources):
T.assert_in(expected_path['name'],
self.data_source.key_mapping_cache)
T.assert_equal(
self.data_source.key_mapping_cache[expected_path['name']],
expected_data_source)
def test_list_path_no_path_duplicates(self):
"""Tests that when no path is specified the correct results are returned
and the repeat key is not cached with the wrong data source.
"""
test_path = None
expected_paths = [{
'name': 'src.MajorSource%d' % i,
'type': 'dir'
} for i in xrange(len(self.data_source.data_sources))]
with self._mock_ds_method(
'_request_paths_from_ds') as mock_request_paths:
mock_request_path_list = [[path] for path in expected_paths]
mock_request_path_list[-1].append({
'name': 'src.MajorSource1',
'type': 'dir'
})
mock_request_paths.side_effect = mock_request_path_list
actual_paths = self.data_source.list_path(test_path)
T.assert_equal(mock_request_paths.call_count,
len(self.data_source.data_sources))
for ds in self.data_source.data_sources:
mock_request_paths.assert_any_call(ds, test_path)
T.assert_equal(expected_paths, actual_paths)
for expected_path, expected_data_source in zip(
expected_paths, self.data_source.data_sources):
T.assert_in(expected_path['name'],
self.data_source.key_mapping_cache)
T.assert_equal(
self.data_source.key_mapping_cache[expected_path['name']],
expected_data_source)
def test_list_path(self):
"""Tests the behavior of list_path when a path is specified."""
test_path = ['src.MajorSource1']
expected_paths = [{
'name': 'src.minorSource%d' % i,
'type': 'dir'
} for i in xrange(5)]
with self._mock_ds_method(
'_request_paths_from_ds') as mock_request_paths:
with self._mock_ds_method(
'_data_source_for_stat_key') as mock_ds_for_stat_key:
mock_ds_for_stat_key.return_value = self.data_source.data_sources[
1]
mock_request_paths.return_value = [
path for path in expected_paths
]
actual_paths = self.data_source.list_path(test_path)
mock_ds_for_stat_key.assert_called_once_with(test_path[0])
mock_request_paths.assert_called_once_with(
self.data_source.data_sources[1], test_path)
T.assert_equal(expected_paths, actual_paths)
def test_data_single_from_single_source(self):
"""Tests the behavior of data when asking for a single source."""
test_data_source = self.data_source.data_sources[2]
test_stat = [[
'src.MajorSourceA', 'key.minorStat1', 'variant.Blue',
'50th_percentile'
]]
test_start = 100
test_end = 200
test_width = 500
dummy_data = [{
't': 500,
'v': [120]
}, {
't': 505,
'v': [140]
}, {
't': 510,
'v': [160]
}]
expected_data = json.dumps(dummy_data)
with self._mock_ds_method(
'_data_source_for_stat_key') as mock_ds_for_stat_key:
with self._mock_ds_method(
'_request_data_from_ds') as mock_request_data:
mock_ds_for_stat_key.return_value = test_data_source
mock_request_data.return_value = dummy_data
actual_data = self.data_source.data(test_stat, test_start,
test_end, test_width)
mock_ds_for_stat_key.assert_called_once_with(test_stat[0][0])
mock_request_data.assert_called_once_with(
test_data_source, test_stat, test_start, test_end,
test_width)
T.assert_equal(expected_data, actual_data)
def test_data_multiple_from_single_source(self):
"""Tests the behavior of data when asking for multiple stats from a
single source.
"""
test_data_source = self.data_source.data_sources[2]
test_stats = [[
'src.MajorSourceA', 'key.minorStat1', 'variant.Blue',
'50th_percentile'
],
[
'src.MajorSourceA', 'key.minorStat1', 'variant.Red',
'75th_percentile'
]]
test_start = 100
test_end = 200
test_width = 500
dummy_data = [{
't': 500,
'v': [120, 190]
}, {
't': 505,
'v': [140, 200]
}, {
't': 510,
'v': [160, 40]
}]
expected_data = json.dumps(dummy_data)
with self._mock_ds_method(
'_data_source_for_stat_key') as mock_ds_for_stat_key:
with self._mock_ds_method(
'_request_data_from_ds') as mock_request_data:
mock_ds_for_stat_key.side_effect = [
test_data_source, test_data_source
]
mock_request_data.return_value = dummy_data
actual_data = self.data_source.data(test_stats, test_start,
test_end, test_width)
T.assert_equal(mock_ds_for_stat_key.call_count,
len(test_stats))
for stat in test_stats:
mock_ds_for_stat_key.assert_any_call(stat[0])
mock_request_data.assert_called_once_with(
test_data_source, test_stats, test_start, test_end,
test_width)
T.assert_equal(expected_data, actual_data)
def test_data_multiple_from_multiple_sources(self):
"""Tests the behavior of data when asking for multiple stats from
multiple sources.
"""
# A -> 0, B -> 1, C -> 2 in self.data_source.data_sources
test_stats = [[
'src.MajorSourceA', 'key.minorStat1', 'variant.Blue',
'50th_percentile'
], [
'src.MajorSourceB', 'key.minorStatGolf', 'variant.eskimo', 'count'
],
[
'src.MajorSourceA', 'key.minorStat1', 'variant.Red',
'75th_percentile'
],
[
'src.MajorSourceC', 'key.minorStatPidgeon',
'variant.alice', 'dart'
]]
test_start = 100
test_end = 200
test_width = 500
# Lists data in the order of the data sources that will be asked
dummy_data_list = [[{
't': 500,
'v': [120, 190]
}, {
't': 505,
'v': [140, 200]
}, {
't': 510,
'v': [160, 40]
}],
[{
't': 496,
'v': [90]
}, {
't': 499,
'v': [93]
}, {
't': 505,
'v': [101]
}, {
't': 510,
'v': [100]
}],
[{
't': 500,
'v': [4]
}, {
't': 507,
'v': [10]
}, {
't': 510,
'v': [4]
}, {
't': 512,
'v': [8]
}]]
# Derived from dummy_data_list
expected_data = json.dumps([{
't': 496,
'v': [None, 90, None, None]
}, {
't': 499,
'v': [None, 93, None, None]
}, {
't': 500,
'v': [120, None, 190, 4]
}, {
't': 505,
'v': [140, 101, 200, None]
}, {
't': 507,
'v': [None, None, None, 10]
}, {
't': 510,
'v': [160, 100, 40, 4]
}, {
't': 512,
'v': [None, None, None, 8]
}])
with self._mock_ds_method(
'_data_source_for_stat_key') as mock_ds_for_stat_key:
with self._mock_ds_method(
'_request_data_from_ds') as mock_request_data:
mock_ds_for_stat_key.side_effect = [
self.data_source.data_sources[0],
self.data_source.data_sources[1],
self.data_source.data_sources[0],
self.data_source.data_sources[2],
]
mock_request_data.side_effect = dummy_data_list
actual_data = self.data_source.data(test_stats, test_start,
test_end, test_width)
T.assert_equal(mock_ds_for_stat_key.call_count,
len(test_stats))
for stat in test_stats:
mock_ds_for_stat_key.assert_any_call(stat[0])
T.assert_equal(mock_request_data.call_count, 3)
mock_request_data.assert_any_call(
self.data_source.data_sources[0],
[test_stats[0], test_stats[2]], test_start, test_end,
test_width)
mock_request_data.assert_any_call(
self.data_source.data_sources[1], [test_stats[1]],
test_start, test_end, test_width)
mock_request_data.assert_any_call(
self.data_source.data_sources[2], [test_stats[3]],
test_start, test_end, test_width)
T.assert_equal(expected_data, actual_data)
def test_int_dict_to_list(self):
"""Just a quick test for the positive path of _int_dict_to_list."""
test_dict = {1: 'a', 2: 'b', 3: ['c', {'a': 'i'}], 4: 0, 11: 'cya'}
expected_list = ['a', 'b', ['c', {'a': 'i'}], 0, 'cya']
actual_list = self.data_source._int_dict_to_list(test_dict)
T.assert_equal(expected_list, actual_list)
def test_request_paths_from_ds(self):
"""Tests the behavior of _request_paths_from_ds when a path is
specified.
"""
test_data_source = self.data_source.data_sources[0]
test_path = ['src.A']
mock_paths = ['key.D', 'key.E', 'key.F']
expected_ask_path = json.dumps([test_data_source['data_source_hash']] +
test_path)
expected_paths = mock_paths
expected_token = util.generate_access_token(
test_data_source['secret_key'])
with self._patch_urlopen() as mock_urlopen:
mock_urlopen.return_value = StringIO(json.dumps(mock_paths))
actual_paths = self.data_source._request_paths_from_ds(
test_data_source, test_path)
T.assert_equal(mock_urlopen.call_count, 1)
T.assert_equal(mock_paths, actual_paths) # 'cause why not?
T.assert_equal(len(mock_urlopen.call_args[0]), 1)
submitted_url = mock_urlopen.call_args[0][0]
parsed_url = urlparse.urlparse(submitted_url)
query_params = urlparse.parse_qs(parsed_url.query)
T.assert_equal(parsed_url.scheme, 'http')
T.assert_equal(parsed_url.path, '/sources')
T.assert_equal(parsed_url.fragment, '')
T.assert_equal(query_params['path'], [expected_ask_path])
T.assert_equal(query_params['token'], [expected_token])
T.assert_equal(actual_paths, expected_paths)
def test_request_paths_from_ds_no_path(self):
"""Tests the behavior of _request_paths_from_ds when no path is
specified."""
test_data_source = self.data_source.data_sources[0]
test_path = None
mock_paths = ['src.A', 'src.B', 'src.C']
expected_ask_path = json.dumps([test_data_source['data_source_hash']])
expected_paths = mock_paths
expected_token = util.generate_access_token(
test_data_source['secret_key'])
with self._patch_urlopen() as mock_urlopen:
mock_urlopen.return_value = StringIO(json.dumps(mock_paths))
actual_paths = self.data_source._request_paths_from_ds(
test_data_source, test_path)
T.assert_equal(mock_urlopen.call_count, 1)
T.assert_equal(mock_paths, actual_paths) # 'cause why not?
T.assert_equal(len(mock_urlopen.call_args[0]), 1)
submitted_url = mock_urlopen.call_args[0][0]
parsed_url = urlparse.urlparse(submitted_url)
query_params = urlparse.parse_qs(parsed_url.query)
T.assert_equal(parsed_url.scheme, 'http')
T.assert_equal(parsed_url.path, '/sources')
T.assert_equal(parsed_url.fragment, '')
T.assert_equal(query_params['path'], [expected_ask_path])
T.assert_equal(query_params['token'], [expected_token])
T.assert_equal(actual_paths, expected_paths)
def test_request_paths_from_ds_url_error(self):
"""Tests that _request_paths_from_ds returns an empty list if it
can't contact the data_server.
"""
test_data_source = self.data_source.data_sources[0]
test_path = None
expected_paths = []
with self._patch_urlopen() as mock_urlopen:
with self._mock_ds_method('logger') as mock_logger:
# Set urlopen raise a URLError
mock_urlopen.side_effect = urllib2.URLError(
"Couldn't talk to the data server")
actual_paths = self.data_source._request_paths_from_ds(
test_data_source, test_path)
T.assert_equal(mock_urlopen.call_count, 1)
T.assert_equal(actual_paths, expected_paths)
mock_logger.exception.assert_called_once_with(
"Failed to fetch paths for %s from %s" %
(test_path, test_data_source['data_server_url']))
def test_request_paths_from_ds_invalid_response(self):
"""Tests that _request_paths_from_ds returns an empty list if it
can't parse the result of its interaction with the data server.
"""
test_data_source = self.data_source.data_sources[0]
test_path = None
expected_paths = []
with self._patch_urlopen() as mock_urlopen:
with self._mock_ds_method('logger') as mock_logger:
# Set urlopen to return a result that can't be JSON parsed
mock_urlopen.return_value = StringIO("[{{1_\]")
actual_paths = self.data_source._request_paths_from_ds(
test_data_source, test_path)
T.assert_equal(mock_urlopen.call_count, 1)
T.assert_equal(actual_paths, expected_paths)
mock_logger.exception.assert_called_once_with(
"Invalid response received from %s" %
test_data_source['data_server_url'])
def test_request_data_from_ds(self):
"""Checks that _request_data_from_ds knows how to ask for data from
other data servers correctly.
TODO(fhats): This can certainly benefit from some integration tests.
"""
test_data_source = self.data_source.data_sources[0]
test_sources = [['src.EndpointTIming', 'stat.A', 'variant.logged_in'],
['src.ErrorCount', 'stat.B', 'variant.logged_out']]
test_start = 100
test_end = 200
test_width = 30
expected_sources = map(
lambda x: [test_data_source['data_source_hash']] + x, test_sources)
expected_token = util.generate_access_token(
test_data_source['secret_key'])
with self._patch_urlopen() as mock_urlopen:
# Just sprinkle some mock data to return
mock_data = [{'t': 150, 'v': [5, 6]}, {'t': 160, 'v': [9, 10]}]
mock_urlopen.return_value = StringIO(json.dumps(mock_data))
returned_data = self.data_source._request_data_from_ds(
test_data_source, test_sources, test_start, test_end,
test_width)
T.assert_equal(mock_urlopen.call_count, 1)
T.assert_equal(mock_data, returned_data) # 'cause why not?
T.assert_equal(len(mock_urlopen.call_args[0]), 1)
submitted_url = mock_urlopen.call_args[0][0]
parsed_url = urlparse.urlparse(submitted_url)
query_params = urlparse.parse_qs(parsed_url.query)
T.assert_equal(parsed_url.scheme, 'http')
T.assert_equal(parsed_url.path, '/data')
T.assert_equal(parsed_url.fragment, '')
T.assert_equal(query_params['sources'],
[json.dumps(expected_sources)])
T.assert_equal(query_params['start'], [str(test_start)])
T.assert_equal(query_params['end'], [str(test_end)])
T.assert_equal(query_params['width'], [str(test_width)])
T.assert_equal(query_params['token'], [expected_token])
def test_request_data_from_ds_urlerror(self):
test_data_source = self.data_source.data_sources[0]
test_sources = [['src.EndpointTIming', 'stat.A', 'variant.logged_in'],
['src.ErrorCount', 'stat.B', 'variant.logged_out']]
test_start = 100
test_end = 200
test_width = 30
with self._patch_urlopen() as mock_urlopen:
mock_urlopen.side_effect = urllib2.URLError('fake URLError')
returned_data = self.data_source._request_data_from_ds(
test_data_source, test_sources, test_start, test_end,
test_width)
T.assert_equal(mock_urlopen.call_count, 1)
T.assert_equal(len(mock_urlopen.call_args[0]), 1)
T.assert_equal(returned_data, [])
def test_data_source_for_stat_key_in_cache(self):
"""Tests that _data_source_for_stat_key uses the key_mapping_cache if
it can.
"""
expected_data_source = self.data_source.data_sources[2]
stat_key = 'src.test_stat_key'
self.data_source.key_mapping_cache[stat_key] = expected_data_source
with self._mock_ds_method(
'_find_data_source_for_stat_key') as mock_find_ds_for_stat:
actual_ds = self.data_source._data_source_for_stat_key(stat_key)
T.assert_equal(mock_find_ds_for_stat.call_count, 0)
T.assert_equal(actual_ds, expected_data_source)
def test_data_source_for_stat_key_not_cached(self):
"""Tests that _data_source_for_stat_key asks _find_data_source_for_stat_key
if the stat key isn't cached in the key_mapping_cache.
"""
expected_data_source = self.data_source.data_sources[2]
stat_key = 'src.test_stat_key'
with self._mock_ds_method(
'_find_data_source_for_stat_key') as mock_find_ds_for_stat:
mock_find_ds_for_stat.return_value = expected_data_source
actual_ds = self.data_source._data_source_for_stat_key(stat_key)
mock_find_ds_for_stat.assert_called_once_with(stat_key)
T.assert_equal(actual_ds, expected_data_source)
def test_find_data_source_for_stat_key(self):
"""Tests _find_data_source_for_stat_key when it's provided by one of
the configured data sources.
"""
expected_data_source = {
'data_server_url': "http://b.com",
'data_source_hash': util.generate_ds_key("another.data.source"),
'secret_key': "TEST_SECRET_TWO"
}
test_key = 'src.our_key'
def fake_paths_from_ds(data_source, path):
if data_source == expected_data_source:
return [
{
"name": test_key
},
]
else:
return [
{
"name": "src.not_our_key"
},
]
with mock.patch.object(self.data_source, '_request_paths_from_ds',
fake_paths_from_ds):
actual_ds = self.data_source._find_data_source_for_stat_key(
test_key)
T.assert_equal(expected_data_source, actual_ds)
T.assert_in(test_key, self.data_source.key_mapping_cache)
T.assert_equal(expected_data_source,
self.data_source.key_mapping_cache[test_key])
def test_find_data_source_for_stat_key_not_found(self):
"""Tests _find_data_source_for_stat_key's behavior when the stat_key
isn't made available by any data sources.
"""
expected_data_source = None
test_key = 'src.some_test_stat'
with self._mock_ds_method(
'_request_paths_from_ds') as mock_request_paths:
mock_request_paths.return_value = [{
"name": "a"
}, {
"name": "b"
}, {
"name": "c"
}]
actual_ds = self.data_source._find_data_source_for_stat_key(
test_key)
T.assert_equal(mock_request_paths.call_count,
len(self.test_data_sources))
for ds in self.data_source.data_sources:
mock_request_paths.assert_any_call(ds, None)
T.assert_equal(expected_data_source, actual_ds)
@contextmanager
def _patch_urlopen(self):
with mock.patch("firefly.data_sources.aggregating_data_source.urlopen"
) as mock_urlopen:
yield mock_urlopen
@contextmanager
def _mock_ds_method(self, method):
with mock.patch.object(self.data_source, method) as mock_thing:
yield mock_thing