def test_v3_index(self): """ Test reading v3 tile indexes """ env = Environment.instance() env.config = Mock() env.config.TILE_INDEX_FILES = {"desktop": "some_url"} v3_data = {"STAR/en-US": {"legacy": "data"}, "__ver__": 3} class TestResponse: status_code = 200 def json(self): return v3_data onyx.environment.grequests.map = Mock(return_value=[TestResponse()]) index_mock = MagicMock() env.config.LINKS_LOCALIZATIONS = index_mock gevent.spawn(_read_tile_index_loop, env) gevent.sleep(0) # make the event loop tick index_mock.__setitem__.assert_called_with("desktop", v3_data)
def test_index_request_server_error(self): """ Test request server error """ env = Environment.instance() env.config = Mock() env.config.TILE_INDEX_FILES = {'desktop': 'some_url'} env.log_dict = Mock() class TestResponse: status_code = 500 def json(self): assert (False) onyx.environment.grequests.map = Mock(return_value=[TestResponse()]) index_mock = MagicMock() env.config.LINKS_LOCALIZATIONS = index_mock gevent.spawn(_read_tile_index_loop, env) gevent.sleep(0) # make the event loop tick assert_equals(0, index_mock.call_count) env.log_dict.assert_any_calls() assert_equals('gevent_tiles_server_update_error', env.log_dict.call_args[1]['action'])
def test_index_request_server_error(self): """ Test request server error """ env = Environment.instance() env.config = Mock() env.config.TILE_INDEX_FILES = {"desktop": "some_url"} env.log_dict = Mock() class TestResponse: status_code = 500 def json(self): assert False onyx.environment.grequests.map = Mock(return_value=[TestResponse()]) index_mock = MagicMock() env.config.LINKS_LOCALIZATIONS = index_mock gevent.spawn(_read_tile_index_loop, env) gevent.sleep(0) # make the event loop tick assert_equals(0, index_mock.call_count) env.log_dict.assert_any_calls() assert_equals("gevent_tiles_server_update_error", env.log_dict.call_args[1]["action"])
def test_index_failure(self): """ Test json file read failure """ env = Environment.instance() test_file = NamedTemporaryFile() env.config = Mock() env.config.TILE_INDEX_FILES = {'desktop': test_file.name} env.log_dict = Mock() v3_data = "{'STAR/en-US': {'legacy': 'data'}, '__ver__': 3" test_file.write(v3_data) test_file.flush() index_mock = MagicMock() env.config.LINKS_LOCALIZATIONS = index_mock gevent.spawn(_read_tile_index_loop, env) gevent.sleep(0) # make the event loop tick assert_equals(0, index_mock.call_count) env.log_dict.assert_any_calls() assert_equals('gevent_tiles_update_error', env.log_dict.call_args[1]['action'])
def test_v3_index(self): """ Test reading v3 tile indexes """ env = Environment.instance() env.config = Mock() env.config.TILE_INDEX_FILES = {'desktop': 'some_url'} v3_data = {'STAR/en-US': {'legacy': 'data'}, '__ver__': 3} class TestResponse: status_code = 200 def json(self): return v3_data onyx.environment.grequests.map = Mock(return_value=[TestResponse()]) index_mock = MagicMock() env.config.LINKS_LOCALIZATIONS = index_mock gevent.spawn(_read_tile_index_loop, env) gevent.sleep(0) # make the event loop tick index_mock.__setitem__.assert_called_with('desktop', v3_data)
def test_index_request_failure(self): """ Test request failure """ env = Environment.instance() env.config = Mock() env.config.TILE_INDEX_FILES = {'desktop': 'some_url'} env.log_dict = Mock() map_mock = Mock() map_mock.side_effect = Exception('some grequests error') onyx.environment.grequests.map = map_mock index_mock = MagicMock() env.config.LINKS_LOCALIZATIONS = index_mock gevent.spawn(_read_tile_index_loop, env) gevent.sleep(0) # make the event loop tick assert_equals(0, index_mock.call_count) env.log_dict.assert_any_calls() assert_equals('gevent_tiles_request_error', env.log_dict.call_args[1]['action'])
def flood_user_event(num_msgs, min_kb, *args, **kwargs): """ Flood the user event log with simulated data """ ua_str = 'Onyx-LogFlood/1.0' ip = '0.0.0.0' locale = 'en-US' ver = '2' min_bytes = min_kb * 1024 url_suffix = '.mozilla.org' num_messages_for_size = None events = [] def gen_tiles(num_urls=12): for i in xrange(num_urls): url_prefix = str(uuid.uuid4()) url = '{0}{1}'.format(url_prefix, url_suffix) id = random.randint(1, 500) yield {'url': url, 'id': id} for i in xrange(num_msgs): tiles = [] message = { 'ip': ip, 'ua': ua_str, 'locale': locale, 'ver': ver, 'tiles': tiles, 'view': len(tiles) } bytes_message = len(json.dumps(message)) if num_messages_for_size is None: # find and cache the number of messages to reach min_bytes num_messages = 0 while bytes_message < min_bytes: for i in gen_tiles(10): tiles.append(i) message['view'] = len(tiles) bytes_message = len(json.dumps(message)) num_messages += 10 num_messages_for_size = num_messages else: tiles.extend(gen_tiles(num_messages_for_size)) message['view'] = len(tiles) events.append(message) env = Environment.instance() for event in events: env.log_dict(name='user_event', message=event)
def test_index_request_correctness(self): """ Test requests are correct and executed in the right order """ env = Environment.instance() key_mirror = { 'desktop': 'http://desktop.url', 'desktop-prerelease': 'http://desktop-prerelease.url', 'android': 'http://android.url', 'hello': 'http://hello.url' } env.config = Mock() env.config.TILE_INDEX_FILES = key_mirror env.log_dict = Mock() class TestResponse: status_code = 200 def __init__(self, url): self.url = url def json(self): # instead of JSON data, return a URL return self.url def map_response(*args, **kwargs): urls = args[0] output = [] for url in urls: output.append(TestResponse(url)) return output def get_response(*args, **kwargs): return args[0] onyx.environment.grequests.map = Mock(side_effect=map_response) onyx.environment.grequests.get = Mock(side_effect=get_response) env.config.LINKS_LOCALIZATIONS = { 'desktop': None, 'desktop-prerelease': None, 'android': None, 'hello': None } gevent.spawn(_read_tile_index_loop, env) gevent.sleep(0) # make the event loop tick env.log_dict.assert_no_calls() assert_equals(key_mirror, env.config.LINKS_LOCALIZATIONS)
def test_index_request_correctness(self): """ Test requests are correct and executed in the right order """ env = Environment.instance() key_mirror = { "desktop": "http://desktop.url", "desktop-prerelease": "http://desktop-prerelease.url", "android": "http://android.url", "hello": "http://hello.url", } env.config = Mock() env.config.TILE_INDEX_FILES = key_mirror env.log_dict = Mock() class TestResponse: status_code = 200 def __init__(self, url): self.url = url def json(self): # instead of JSON data, return a URL return self.url def map_response(*args, **kwargs): urls = args[0] output = [] for url in urls: output.append(TestResponse(url)) return output def get_response(*args, **kwargs): return args[0] onyx.environment.grequests.map = Mock(side_effect=map_response) onyx.environment.grequests.get = Mock(side_effect=get_response) env.config.LINKS_LOCALIZATIONS = {"desktop": None, "desktop-prerelease": None, "android": None, "hello": None} gevent.spawn(_read_tile_index_loop, env) gevent.sleep(0) # make the event loop tick env.log_dict.assert_no_calls() assert_equals(key_mirror, env.config.LINKS_LOCALIZATIONS)
def test_v3_index(self): """ Test reading v3 tile indexes """ env = Environment.instance() test_file = NamedTemporaryFile() env.config = Mock() env.config.TILE_INDEX_FILES = {'desktop': test_file.name} v3_data = {'STAR/en-US': {'legacy': 'data'}, '__ver__': 3} json.dump(v3_data, test_file) test_file.flush() index_mock = MagicMock() env.config.LINKS_LOCALIZATIONS = index_mock gevent.spawn(_read_tile_index_loop, env) gevent.sleep(0) # make the event loop tick index_mock.__setitem__.assert_called_with('desktop', v3_data)
def environment_manager_create(config=None): """ Create and configure application If not specified, the default config will be loaded. If the ONYX_SETTINGS is provided, the file location will take precedence """ for path in CONFIG_PATH_LOCATIONS: sys.path.append(path) if config is None: # None will be passed by manage.py. # A default param value will get overwritten, so this is implemented here. config = 'onyx.default_settings.DefaultConfig' config = os.environ.get('ONYX_SETTINGS', config) env = Environment.instance(config) from onyx.webapp import setup_routes setup_routes(env.application) return env.application
def test_index_request_failure(self): """ Test request failure """ env = Environment.instance() env.config = Mock() env.config.TILE_INDEX_FILES = {"desktop": "some_url"} env.log_dict = Mock() map_mock = Mock() map_mock.side_effect = Exception("some grequests error") onyx.environment.grequests.map = map_mock index_mock = MagicMock() env.config.LINKS_LOCALIZATIONS = index_mock gevent.spawn(_read_tile_index_loop, env) gevent.sleep(0) # make the event loop tick assert_equals(0, index_mock.call_count) env.log_dict.assert_any_calls() assert_equals("gevent_tiles_request_error", env.log_dict.call_args[1]["action"])
import random import ujson from flask import (Blueprint, request, make_response, Response, redirect) from onyx.environment import Environment from onyx.api.v2 import handle_ping links = Blueprint('v3_links', __name__, url_prefix='/v3/links') env = Environment.instance() channels_valid = { 'esr', 'release', 'beta', 'aurora', 'nightly', 'android', 'hello' } channels_prerelease = {'beta', 'aurora', 'nightly'} channels_release = {'esr', 'release'} @links.route('/fetch/<locale>/<channel>', methods=['GET']) @env.statsd.timer('v3_links_fetch') def fetch(locale=None, channel=None): """ Given a locale, return locale-specific links if possible. """ ip_addrs = None ip_addr = None ua = None ip_addrs = request.headers.get('X-Forwarded-For') if ip_addrs is None: ip_addrs = request.remote_addr if ip_addrs is not None:
def create_app(self): DefaultConfig.ENVIRONMENT = 'test' self.app = environment_manager_create() self.env = Environment.instance() self.env.config.LINKS_LOCALIZATIONS = DEFAULT_DATA return self.app
import logging import random from flask import ( Blueprint, request, make_response, redirect, Response ) from onyx.environment import Environment import ujson links = Blueprint('v2_links', __name__, url_prefix='/v2/links') env = Environment.instance() @links.route('/fetch', methods=['POST']) @links.route('/fetch/<locale>', methods=['GET']) @env.statsd.timer('v2_links_fetch') def fetch(locale=None): """ Given a locale, return locale-specific links if possible. """ ip_addrs = None ip_addr = None ua = None try: ip_addrs = request.headers.get('X-Forwarded-For') if ip_addrs is None: