def test_v3_index(self):
"""
Test reading v3 tile indexes
"""
env = Environment.instance()
env.config = Mock()
env.config.TILE_INDEX_FILES = {"desktop": "some_url"}
v3_data = {"STAR/en-US": {"legacy": "data"}, "__ver__": 3}
class TestResponse:
status_code = 200
def json(self):
return v3_data
onyx.environment.grequests.map = Mock(return_value=[TestResponse()])
index_mock = MagicMock()
env.config.LINKS_LOCALIZATIONS = index_mock
gevent.spawn(_read_tile_index_loop, env)
gevent.sleep(0) # make the event loop tick
index_mock.__setitem__.assert_called_with("desktop", v3_data)
def test_index_request_server_error(self):
"""
Test request server error
"""
env = Environment.instance()
env.config = Mock()
env.config.TILE_INDEX_FILES = {'desktop': 'some_url'}
env.log_dict = Mock()
class TestResponse:
status_code = 500
def json(self):
assert (False)
onyx.environment.grequests.map = Mock(return_value=[TestResponse()])
index_mock = MagicMock()
env.config.LINKS_LOCALIZATIONS = index_mock
gevent.spawn(_read_tile_index_loop, env)
gevent.sleep(0) # make the event loop tick
assert_equals(0, index_mock.call_count)
env.log_dict.assert_any_calls()
assert_equals('gevent_tiles_server_update_error',
env.log_dict.call_args[1]['action'])
def test_index_request_server_error(self):
"""
Test request server error
"""
env = Environment.instance()
env.config = Mock()
env.config.TILE_INDEX_FILES = {"desktop": "some_url"}
env.log_dict = Mock()
class TestResponse:
status_code = 500
def json(self):
assert False
onyx.environment.grequests.map = Mock(return_value=[TestResponse()])
index_mock = MagicMock()
env.config.LINKS_LOCALIZATIONS = index_mock
gevent.spawn(_read_tile_index_loop, env)
gevent.sleep(0) # make the event loop tick
assert_equals(0, index_mock.call_count)
env.log_dict.assert_any_calls()
assert_equals("gevent_tiles_server_update_error", env.log_dict.call_args[1]["action"])
def test_index_failure(self):
"""
Test json file read failure
"""
env = Environment.instance()
test_file = NamedTemporaryFile()
env.config = Mock()
env.config.TILE_INDEX_FILES = {'desktop': test_file.name}
env.log_dict = Mock()
v3_data = "{'STAR/en-US': {'legacy': 'data'}, '__ver__': 3"
test_file.write(v3_data)
test_file.flush()
index_mock = MagicMock()
env.config.LINKS_LOCALIZATIONS = index_mock
gevent.spawn(_read_tile_index_loop, env)
gevent.sleep(0) # make the event loop tick
assert_equals(0, index_mock.call_count)
env.log_dict.assert_any_calls()
assert_equals('gevent_tiles_update_error', env.log_dict.call_args[1]['action'])
def test_v3_index(self):
"""
Test reading v3 tile indexes
"""
env = Environment.instance()
env.config = Mock()
env.config.TILE_INDEX_FILES = {'desktop': 'some_url'}
v3_data = {'STAR/en-US': {'legacy': 'data'}, '__ver__': 3}
class TestResponse:
status_code = 200
def json(self):
return v3_data
onyx.environment.grequests.map = Mock(return_value=[TestResponse()])
index_mock = MagicMock()
env.config.LINKS_LOCALIZATIONS = index_mock
gevent.spawn(_read_tile_index_loop, env)
gevent.sleep(0) # make the event loop tick
index_mock.__setitem__.assert_called_with('desktop', v3_data)
def test_index_request_failure(self):
"""
Test request failure
"""
env = Environment.instance()
env.config = Mock()
env.config.TILE_INDEX_FILES = {'desktop': 'some_url'}
env.log_dict = Mock()
map_mock = Mock()
map_mock.side_effect = Exception('some grequests error')
onyx.environment.grequests.map = map_mock
index_mock = MagicMock()
env.config.LINKS_LOCALIZATIONS = index_mock
gevent.spawn(_read_tile_index_loop, env)
gevent.sleep(0) # make the event loop tick
assert_equals(0, index_mock.call_count)
env.log_dict.assert_any_calls()
assert_equals('gevent_tiles_request_error',
env.log_dict.call_args[1]['action'])
def flood_user_event(num_msgs, min_kb, *args, **kwargs):
"""
Flood the user event log with simulated data
"""
ua_str = 'Onyx-LogFlood/1.0'
ip = '0.0.0.0'
locale = 'en-US'
ver = '2'
min_bytes = min_kb * 1024
url_suffix = '.mozilla.org'
num_messages_for_size = None
events = []
def gen_tiles(num_urls=12):
for i in xrange(num_urls):
url_prefix = str(uuid.uuid4())
url = '{0}{1}'.format(url_prefix, url_suffix)
id = random.randint(1, 500)
yield {'url': url, 'id': id}
for i in xrange(num_msgs):
tiles = []
message = {
'ip': ip,
'ua': ua_str,
'locale': locale,
'ver': ver,
'tiles': tiles,
'view': len(tiles)
}
bytes_message = len(json.dumps(message))
if num_messages_for_size is None:
# find and cache the number of messages to reach min_bytes
num_messages = 0
while bytes_message < min_bytes:
for i in gen_tiles(10):
tiles.append(i)
message['view'] = len(tiles)
bytes_message = len(json.dumps(message))
num_messages += 10
num_messages_for_size = num_messages
else:
tiles.extend(gen_tiles(num_messages_for_size))
message['view'] = len(tiles)
events.append(message)
env = Environment.instance()
for event in events:
env.log_dict(name='user_event', message=event)
def flood_user_event(num_msgs, min_kb, *args, **kwargs):
"""
Flood the user event log with simulated data
"""
ua_str = 'Onyx-LogFlood/1.0'
ip = '0.0.0.0'
locale = 'en-US'
ver = '2'
min_bytes = min_kb * 1024
url_suffix = '.mozilla.org'
num_messages_for_size = None
events = []
def gen_tiles(num_urls=12):
for i in xrange(num_urls):
url_prefix = str(uuid.uuid4())
url = '{0}{1}'.format(url_prefix, url_suffix)
id = random.randint(1, 500)
yield {'url': url, 'id': id}
for i in xrange(num_msgs):
tiles = []
message = {
'ip': ip,
'ua': ua_str,
'locale': locale,
'ver': ver,
'tiles': tiles,
'view': len(tiles)
}
bytes_message = len(json.dumps(message))
if num_messages_for_size is None:
# find and cache the number of messages to reach min_bytes
num_messages = 0
while bytes_message < min_bytes:
for i in gen_tiles(10):
tiles.append(i)
message['view'] = len(tiles)
bytes_message = len(json.dumps(message))
num_messages += 10
num_messages_for_size = num_messages
else:
tiles.extend(gen_tiles(num_messages_for_size))
message['view'] = len(tiles)
events.append(message)
env = Environment.instance()
for event in events:
env.log_dict(name='user_event', message=event)
def test_index_request_correctness(self):
"""
Test requests are correct and executed in the right order
"""
env = Environment.instance()
key_mirror = {
'desktop': 'http://desktop.url',
'desktop-prerelease': 'http://desktop-prerelease.url',
'android': 'http://android.url',
'hello': 'http://hello.url'
}
env.config = Mock()
env.config.TILE_INDEX_FILES = key_mirror
env.log_dict = Mock()
class TestResponse:
status_code = 200
def __init__(self, url):
self.url = url
def json(self):
# instead of JSON data, return a URL
return self.url
def map_response(*args, **kwargs):
urls = args[0]
output = []
for url in urls:
output.append(TestResponse(url))
return output
def get_response(*args, **kwargs):
return args[0]
onyx.environment.grequests.map = Mock(side_effect=map_response)
onyx.environment.grequests.get = Mock(side_effect=get_response)
env.config.LINKS_LOCALIZATIONS = {
'desktop': None,
'desktop-prerelease': None,
'android': None,
'hello': None
}
gevent.spawn(_read_tile_index_loop, env)
gevent.sleep(0) # make the event loop tick
env.log_dict.assert_no_calls()
assert_equals(key_mirror, env.config.LINKS_LOCALIZATIONS)
def test_index_request_correctness(self):
"""
Test requests are correct and executed in the right order
"""
env = Environment.instance()
key_mirror = {
"desktop": "http://desktop.url",
"desktop-prerelease": "http://desktop-prerelease.url",
"android": "http://android.url",
"hello": "http://hello.url",
}
env.config = Mock()
env.config.TILE_INDEX_FILES = key_mirror
env.log_dict = Mock()
class TestResponse:
status_code = 200
def __init__(self, url):
self.url = url
def json(self):
# instead of JSON data, return a URL
return self.url
def map_response(*args, **kwargs):
urls = args[0]
output = []
for url in urls:
output.append(TestResponse(url))
return output
def get_response(*args, **kwargs):
return args[0]
onyx.environment.grequests.map = Mock(side_effect=map_response)
onyx.environment.grequests.get = Mock(side_effect=get_response)
env.config.LINKS_LOCALIZATIONS = {"desktop": None, "desktop-prerelease": None, "android": None, "hello": None}
gevent.spawn(_read_tile_index_loop, env)
gevent.sleep(0) # make the event loop tick
env.log_dict.assert_no_calls()
assert_equals(key_mirror, env.config.LINKS_LOCALIZATIONS)
def test_v3_index(self):
"""
Test reading v3 tile indexes
"""
env = Environment.instance()
test_file = NamedTemporaryFile()
env.config = Mock()
env.config.TILE_INDEX_FILES = {'desktop': test_file.name}
v3_data = {'STAR/en-US': {'legacy': 'data'}, '__ver__': 3}
json.dump(v3_data, test_file)
test_file.flush()
index_mock = MagicMock()
env.config.LINKS_LOCALIZATIONS = index_mock
gevent.spawn(_read_tile_index_loop, env)
gevent.sleep(0) # make the event loop tick
index_mock.__setitem__.assert_called_with('desktop', v3_data)
def environment_manager_create(config=None):
"""
Create and configure application
If not specified, the default config will be loaded.
If the ONYX_SETTINGS is provided, the file location will take precedence
"""
for path in CONFIG_PATH_LOCATIONS:
sys.path.append(path)
if config is None:
# None will be passed by manage.py.
# A default param value will get overwritten, so this is implemented here.
config = 'onyx.default_settings.DefaultConfig'
config = os.environ.get('ONYX_SETTINGS', config)
env = Environment.instance(config)
from onyx.webapp import setup_routes
setup_routes(env.application)
return env.application
def environment_manager_create(config=None):
"""
Create and configure application
If not specified, the default config will be loaded.
If the ONYX_SETTINGS is provided, the file location will take precedence
"""
for path in CONFIG_PATH_LOCATIONS:
sys.path.append(path)
if config is None:
# None will be passed by manage.py.
# A default param value will get overwritten, so this is implemented here.
config = 'onyx.default_settings.DefaultConfig'
config = os.environ.get('ONYX_SETTINGS', config)
env = Environment.instance(config)
from onyx.webapp import setup_routes
setup_routes(env.application)
return env.application
def test_index_request_failure(self):
"""
Test request failure
"""
env = Environment.instance()
env.config = Mock()
env.config.TILE_INDEX_FILES = {"desktop": "some_url"}
env.log_dict = Mock()
map_mock = Mock()
map_mock.side_effect = Exception("some grequests error")
onyx.environment.grequests.map = map_mock
index_mock = MagicMock()
env.config.LINKS_LOCALIZATIONS = index_mock
gevent.spawn(_read_tile_index_loop, env)
gevent.sleep(0) # make the event loop tick
assert_equals(0, index_mock.call_count)
env.log_dict.assert_any_calls()
assert_equals("gevent_tiles_request_error", env.log_dict.call_args[1]["action"])
import random
import ujson
from flask import (Blueprint, request, make_response, Response, redirect)
from onyx.environment import Environment
from onyx.api.v2 import handle_ping
links = Blueprint('v3_links', __name__, url_prefix='/v3/links')
env = Environment.instance()
channels_valid = {
'esr', 'release', 'beta', 'aurora', 'nightly', 'android', 'hello'
}
channels_prerelease = {'beta', 'aurora', 'nightly'}
channels_release = {'esr', 'release'}
@links.route('/fetch/<locale>/<channel>', methods=['GET'])
@env.statsd.timer('v3_links_fetch')
def fetch(locale=None, channel=None):
"""
Given a locale, return locale-specific links if possible.
"""
ip_addrs = None
ip_addr = None
ua = None
ip_addrs = request.headers.get('X-Forwarded-For')
if ip_addrs is None:
ip_addrs = request.remote_addr
if ip_addrs is not None:
def create_app(self):
DefaultConfig.ENVIRONMENT = 'test'
self.app = environment_manager_create()
self.env = Environment.instance()
self.env.config.LINKS_LOCALIZATIONS = DEFAULT_DATA
return self.app
import logging
import random
from flask import (
Blueprint,
request,
make_response,
redirect,
Response
)
from onyx.environment import Environment
import ujson
links = Blueprint('v2_links', __name__, url_prefix='/v2/links')
env = Environment.instance()
@links.route('/fetch', methods=['POST'])
@links.route('/fetch/<locale>', methods=['GET'])
@env.statsd.timer('v2_links_fetch')
def fetch(locale=None):
"""
Given a locale, return locale-specific links if possible.
"""
ip_addrs = None
ip_addr = None
ua = None
try:
ip_addrs = request.headers.get('X-Forwarded-For')
if ip_addrs is None:
def create_app(self):
DefaultConfig.ENVIRONMENT = 'test'
self.app = environment_manager_create()
self.env = Environment.instance()
self.env.config.LINKS_LOCALIZATIONS = DEFAULT_DATA
return self.app