def testInitializeIgnoresTooPositiveLongitude(self): """Ignore a longitude that's above the -180 -> +180 valid range.""" user_defined_latitude = '15.5' user_defined_longitude = '180.1' self.mock_query_params[message.LATITUDE] = user_defined_latitude self.mock_query_params[message.LONGITUDE] = user_defined_longitude query = lookup_query.LookupQuery() query.initialize_from_http_request(self.mock_request) self.assertEqual(self.mock_gae_latitude, query.latitude) self.assertEqual(self.mock_gae_longitude, query.longitude) self.assertEqual(self.mock_gae_city, query.city) self.assertEqual(self.mock_gae_country, query.country)
def testInitializeIgnoresValidUserDefinedLatWithInvalidLon(self): """If lat is valid, but lon is invalid, ignore both.""" user_defined_latitude = '36.0' user_defined_longitude = 'invalid_longitude' self.mock_query_params[message.LATITUDE] = user_defined_latitude self.mock_query_params[message.LONGITUDE] = user_defined_longitude query = lookup_query.LookupQuery() query.initialize_from_http_request(self.mock_request) self.assertEqual(self.mock_gae_latitude, query.latitude) self.assertEqual(self.mock_gae_longitude, query.longitude) self.assertEqual(self.mock_gae_city, query.city) self.assertEqual(self.mock_gae_country, query.country)
def testInitializeIgnoresTooNegativeLatitude(self): """Ignore a latitude that's below the -90 -> +90 valid range.""" user_defined_latitude = '-90.1' user_defined_longitude = '100.2' self.mock_query_params[message.LATITUDE] = user_defined_latitude self.mock_query_params[message.LONGITUDE] = user_defined_longitude query = lookup_query.LookupQuery() query.initialize_from_http_request(self.mock_request) self.assertEqual(self.mock_gae_latitude, query.latitude) self.assertEqual(self.mock_gae_longitude, query.longitude) self.assertEqual(self.mock_gae_city, query.city) self.assertEqual(self.mock_gae_country, query.country)
def testInitializeAcceptsUserDefinedLatLonAndCountry(self): user_defined_latitude = '89.0' user_defined_longitude = '100.0' user_defined_country = 'user_defined_country' self.mock_query_params[message.LATITUDE] = user_defined_latitude self.mock_query_params[message.LONGITUDE] = user_defined_longitude self.mock_query_params[message.COUNTRY] = user_defined_country query = lookup_query.LookupQuery() query.initialize_from_http_request(self.mock_request) self.assertEqual(float(user_defined_latitude), query.latitude) self.assertEqual(float(user_defined_longitude), query.longitude) self.assertIsNone(query.city) self.assertEqual(user_defined_country, query.country)
def testInitializeWhenNoUserDefinedOptionsAreSpecified(self): query = lookup_query.LookupQuery() query.initialize_from_http_request(self.mock_request) self.assertEqual(query.response_format, message.DEFAULT_RESPONSE_FORMAT) self.assertEqual(self.mock_tool_id, query.tool_id) self.assertEqual(message.POLICY_GEO, query.policy) self.assertIsNone(query.metro) self.assertEqual(message.DEFAULT_RESPONSE_FORMAT, query.response_format) self.assertEqual(self.mock_request_ip, query.ip_address) self.assertIsNone(query.tool_address_family) self.assertEqual(self.mock_gae_city, query.city) self.assertEqual(self.mock_gae_country, query.country) self.assertEqual(self.mock_gae_latitude, query.latitude) self.assertEqual(self.mock_gae_longitude, query.longitude) self.assertIsNone(query.distance)
def testInitializeUsesAppEngineGeoDataWhenUserDefinedIpv4MatchesRequestIp( self): # Simulate when the client supplies an explicit IPv4 address in the URL # and it matches the source IP of the web request (i.e. the user # explicitly declared their own IP address). user_defined_ip = self.mock_request_ip self.mock_query_params[message.REMOTE_ADDRESS] = user_defined_ip query = lookup_query.LookupQuery() query.initialize_from_http_request(self.mock_request) self.assertEqual(message.POLICY_GEO, query.policy) self.assertEqual(self.mock_gae_city, query.city) self.assertEqual(self.mock_gae_country, query.country) self.assertEqual(self.mock_gae_latitude, query.latitude) self.assertEqual(self.mock_gae_longitude, query.longitude) # Verify we didn't retrieve any geoip information from Maxmind. self.assertFalse(maxmind.get_ip_geolocation.called)
def testInitializeIgnoresInvalidUserDefinedLatLonEvenIfCityIsValid(self): user_defined_latitude = 'invalid_latitude' user_defined_longitude = 'invalid_longitude' user_defined_city = 'valid_city' self.mock_query_params[message.LATITUDE] = user_defined_latitude self.mock_query_params[message.LONGITUDE] = user_defined_longitude self.mock_query_params[message.CITY] = user_defined_city query = lookup_query.LookupQuery() query.initialize_from_http_request(self.mock_request) # Lat/lon is invalid, user-defined city is ignored if there's no # country to go with it. Treat this as if the request specified nothing. self.assertEqual(message.POLICY_GEO, query.policy) self.assertEqual(self.mock_gae_city, query.city) self.assertEqual(self.mock_gae_country, query.country) self.assertEqual(self.mock_gae_latitude, query.latitude) self.assertEqual(self.mock_gae_longitude, query.longitude)
def testInitializeAcceptsValidUserDefinedCityAndLatLon(self): user_defined_city = 'user_defined_city' user_defined_latitude = '0.0' user_defined_longitude = '4.3' self.mock_query_params[message.CITY] = user_defined_city self.mock_query_params[message.LATITUDE] = user_defined_latitude self.mock_query_params[message.LONGITUDE] = user_defined_longitude query = lookup_query.LookupQuery() query.initialize_from_http_request(self.mock_request) #TODO(mtlynch): We should revisit this because the combination of # parameters here is confusing. What takes precedence if the city # and the lat/lon are contradictory? self.assertEqual(message.POLICY_GEO, query.policy) self.assertEqual(user_defined_city, query.city) self.assertIsNone(query.country) self.assertEqual(float(user_defined_latitude), query.latitude) self.assertEqual(float(user_defined_longitude), query.longitude)
def testInitializeUsesMaxmindWhenAppEngineGeoDataIsMissing(self): # Remove all mock AppEngine headers self.mock_request.headers = {} maxmind_city = 'maxmind_city' maxmind_country = 'maxmind_country' maxmind_latitude = 55.5 maxmind_longitude = 77.7 maxmind.get_ip_geolocation.return_value = maxmind.GeoRecord( city=maxmind_city, country=maxmind_country, latitude=maxmind_latitude, longitude=maxmind_longitude) query = lookup_query.LookupQuery() query.initialize_from_http_request(self.mock_request) self.assertEqual(message.POLICY_GEO, query.policy) self.assertEqual(maxmind_city, query.city) self.assertEqual(maxmind_country, query.country) self.assertEqual(maxmind_latitude, query.latitude) self.assertEqual(maxmind_longitude, query.longitude)
def get(self): """Handles an HTTP GET request. The URL must be in the following format: 'http://mlab-ns.appspot.com/tool-name?query_string', where tool-name is one of the tools running on M-Lab. For more information about the URL and the supported arguments in the query string, see the design doc at http://goo.gl/48S22. """ query = lookup_query.LookupQuery() query.initialize_from_http_request(self.request) logging.info('Policy is %s', query.policy) client_signature = query.calculate_client_signature() lookup_resolver = resolver.new_resolver(query.policy, client_signature) sliver_tools = lookup_resolver.answer_query(query) if sliver_tools is None: return util.send_not_found(self, query.response_format) if query.response_format == message.FORMAT_JSON: self.send_json_response(sliver_tools, query) elif query.response_format == message.FORMAT_HTML: self.send_html_response(sliver_tools, query) elif query.response_format == message.FORMAT_REDIRECT: self.send_redirect_response(sliver_tools, query) elif query.response_format == message.FORMAT_BT: self.send_bt_response(sliver_tools, query) elif query.response_format == message.FORMAT_MAP: candidates = lookup_resolver.get_candidates(query) self.send_map_response(sliver_tools, query, candidates) else: # TODO (claudiu) Discuss what should be the default behaviour. # I think json it's OK since is valid for all tools, while # redirect only applies to web-based tools. self.send_json_response(sliver_tools, query) # TODO (claudiu) Add a FORMAT_TYPE column in the BigQuery schema. self.log_request(query, sliver_tools)
def testAnswerQueryReturnsRandomSubsetWhenQueryIsMissingLatLon(self): """When lat/lon is missing, expect a random subset of tools.""" # TODO(mtlynch): This behavior is confusing because it is inconsistent # with the other resolvers that return None when required attributes are # missing from the query. Change so that all are consistent. query = lookup_query.LookupQuery() query.tool_id = _TOOL_ID mock_fetched_tools = [ _createSliverTool(_TOOL_ID, site_id='abc01', latitude=1.0, longitude=1.0), _createSliverTool(_TOOL_ID, site_id='abc02', latitude=1.0, longitude=1.0), _createSliverTool(_TOOL_ID, site_id='abc03', latitude=1.0, longitude=1.0), _createSliverTool(_TOOL_ID, site_id='abc04', latitude=1.0, longitude=1.0), _createSliverTool(_TOOL_ID, site_id='cba01', latitude=5.0, longitude=5.0) ] # When lat/lon is missing, resolver performs no additional filtering # after fetch filtered_tools_expected = mock_fetched_tools tool_properties_expected = tool_fetcher.ToolProperties( tool_id=_TOOL_ID, status=message.STATUS_ONLINE) self.assertQueryResultMultiToolWithRandomSample( query, mock_fetched_tools, filtered_tools_expected, 4, tool_properties_expected)
def testAnswerQueryWhenMatchingToolsExist(self): query = lookup_query.LookupQuery() query.tool_id = _TOOL_ID mock_fetched_tools = [ _createSliverTool(_TOOL_ID), _createSliverTool(_TOOL_ID) ] # AllResolver should not do any additional filtering on the tools it # fetched. query_results_expected = mock_fetched_tools # Make sure the resolver is fetching only online tools that match the # specified tool ID. tool_properties_expected = sliver_tool_fetcher.ToolProperties( tool_id=_TOOL_ID, status=message.STATUS_ONLINE) self.assertQueryResultMultiTool(query, mock_fetched_tools, query_results_expected, tool_properties_expected)
def testInitializeWithUserDefinedCountryGetsGeolocationForThatCountry(self): user_defined_country = 'user_defined_country' self.mock_query_params[message.COUNTRY] = user_defined_country maxmind_city = None maxmind_country = user_defined_country maxmind_latitude = 55.5 maxmind_longitude = 77.7 maxmind.get_country_geolocation.return_value = maxmind.GeoRecord( city=maxmind_city, country=maxmind_country, latitude=maxmind_latitude, longitude=maxmind_longitude) query = lookup_query.LookupQuery() query.initialize_from_http_request(self.mock_request) self.assertEqual(maxmind_latitude, query.latitude) self.assertEqual(maxmind_longitude, query.longitude) self.assertIsNone(query.city) self.assertEqual(maxmind_country, query.country) self.assertEqual(user_defined_country, query.user_defined_country)
def testAnswerQueryWhenFourToolsAreEquallyClosest(self): """When exactly four tools tie for closest, return those four.""" query = lookup_query.LookupQuery() query.tool_id = _TOOL_ID query.latitude = 0.0 query.longitude = 0.0 mock_fetched_tools = [ _createSliverTool(_TOOL_ID, site_id='abc01', latitude=1.0, longitude=1.0), _createSliverTool(_TOOL_ID, site_id='abc02', latitude=1.0, longitude=1.0), _createSliverTool(_TOOL_ID, site_id='abc03', latitude=1.0, longitude=1.0), _createSliverTool(_TOOL_ID, site_id='abc04', latitude=1.0, longitude=1.0), _createSliverTool(_TOOL_ID, site_id='cba01', latitude=5.0, longitude=5.0) ] # Result should be the four closest tools query_results_expected = mock_fetched_tools[:4] # Make sure the resolver is fetching only online tools that match the # specified tool ID. tool_properties_expected = tool_fetcher.ToolProperties( tool_id=_TOOL_ID, status=message.STATUS_ONLINE) self.assertQueryResultMultiTool(query, mock_fetched_tools, query_results_expected, tool_properties_expected)
def testAnswerQueryWhenMoreThanFourToolsFromDifferentSitesAreEquallyClosest( self): """When more than four tools tie for closest, randomly select four.""" query = lookup_query.LookupQuery() query.tool_id = _TOOL_ID query.latitude = 0.0 query.longitude = 0.0 mock_fetched_tools = [ _createSliverTool( _TOOL_ID, site_id='aaa01', latitude=1.0, longitude=1.0), _createSliverTool( _TOOL_ID, site_id='bbb01', latitude=-1.0, longitude=-1.0), _createSliverTool( _TOOL_ID, site_id='ccc01', latitude=-1.0, longitude=1.0), _createSliverTool( _TOOL_ID, site_id='ddd01', latitude=1.0, longitude=-1.0), _createSliverTool( _TOOL_ID, site_id='eee01', latitude=-1.0, longitude=-1.0), _createSliverTool( _TOOL_ID, site_id='fff01', latitude=-1.0, longitude=1.0), _createSliverTool( _TOOL_ID, site_id='ggg01', latitude=5.0, longitude=5.0) ] # yapf: disable # The mock shuffle reverses the list, so we expect items 2...6 in # reverse order. query_results_expected = mock_fetched_tools[-2:-6:-1] # Make sure the resolver is fetching only online tools that match the # specified tool ID. tool_properties_expected = sliver_tool_fetcher.ToolProperties( tool_id=_TOOL_ID, status=message.STATUS_ONLINE) client_signature_fetcher.ClientSignatureFetcher( ).fetch.return_value = 1.0 self.assertQueryResultWithRandomShuffle(query, mock_fetched_tools, query_results_expected, tool_properties_expected)
def testAnswerQueryWhenMultipleToolsAreEquallyClose(self): """When multiple tools are equally closest, randomly select one.""" query = lookup_query.LookupQuery() query.tool_id = _TOOL_ID query.latitude = 0.0 query.longitude = 0.0 equidistant_tools = (_createSliverTool(_TOOL_ID, site_id='aaa01', latitude=1.0, longitude=5.0), _createSliverTool(_TOOL_ID, site_id='bbb01', latitude=5.0, longitude=1.0)) mock_fetched_tools = [ _createSliverTool(_TOOL_ID, site_id='ccc01', latitude=10.0, longitude=10.0), _createSliverTool(_TOOL_ID, site_id='ddd01', latitude=20.0, longitude=20.0) ] mock_fetched_tools.extend(equidistant_tools) query_results_expected = [equidistant_tools[-1]] tool_properties_expected = sliver_tool_fetcher.ToolProperties( tool_id=_TOOL_ID, status=message.STATUS_ONLINE) client_signature_fetcher.ClientSignatureFetcher( ).fetch.return_value = 1.0 self.assertQueryResultWithRandomShuffle(query, mock_fetched_tools, query_results_expected, tool_properties_expected)
def testAnswerQueryChoosesRandomlyAmongToolsInMetro(self): query = lookup_query.LookupQuery() query.tool_id = _TOOL_ID query.metro = 'aaa' query.tool_address_family = message.ADDRESS_FAMILY_IPv4 mock_fetched_tools = (_createSliverTool(_TOOL_ID, site_id='aaa01'), _createSliverTool(_TOOL_ID, site_id='aaa02'), _createSliverTool(_TOOL_ID, site_id='aaa03')) filtered_tools_expected = mock_fetched_tools # Make sure the resolver is fetching only online tools that match the # specified tool ID in the specified metro. tool_properties_expected = sliver_tool_fetcher.ToolProperties( tool_id=_TOOL_ID, status=message.STATUS_ONLINE, address_family=message.ADDRESS_FAMILY_IPv4, metro=query.metro) self.assertQueryResultSingleToolWithRandomChoice( query, mock_fetched_tools, filtered_tools_expected, tool_properties_expected)
def testAnswerQueryChoosesRandomlyAmongOnlineTools(self): query = lookup_query.LookupQuery() query.tool_id = _TOOL_ID query.tool_address_family = message.ADDRESS_FAMILY_IPv6 mock_fetched_tools = (_createSliverTool(_TOOL_ID, site_id='aaa01'), _createSliverTool(_TOOL_ID, site_id='bbb01'), _createSliverTool(_TOOL_ID, site_id='ccc01'), _createSliverTool(_TOOL_ID, site_id='ddd01')) # Random resolver performs no additional filtering after the fetch. filtered_tools_expected = mock_fetched_tools # Make sure the resolver is fetching only online tools that match the # specified tool ID. tool_properties_expected = sliver_tool_fetcher.ToolProperties( tool_id=_TOOL_ID, address_family=message.ADDRESS_FAMILY_IPv6, status=message.STATUS_ONLINE) self.assertQueryResultSingleToolWithRandomChoice( query, mock_fetched_tools, filtered_tools_expected, tool_properties_expected)
def testInitializeUsesMaxmindWhenUserDefinedIpv6Exists(self): user_defined_ip = '1:2:3::4' self.mock_query_params[message.REMOTE_ADDRESS] = user_defined_ip maxmind_city = 'maxmind_city' maxmind_country = 'maxmind_country' maxmind_latitude = 55.5 maxmind_longitude = 77.7 maxmind.get_ip_geolocation.return_value = maxmind.GeoRecord( city=maxmind_city, country=maxmind_country, latitude=maxmind_latitude, longitude=maxmind_longitude) query = lookup_query.LookupQuery() query.initialize_from_http_request(self.mock_request) # Make sure we looked up the user-defined IP, not the request IP. maxmind.get_ip_geolocation.assert_called_with(user_defined_ip) self.assertEqual(message.POLICY_GEO, query.policy) self.assertEqual(maxmind_city, query.city) self.assertEqual(maxmind_country, query.country) self.assertEqual(maxmind_latitude, query.latitude) self.assertEqual(maxmind_longitude, query.longitude)
def testAnswerQueryWhenMoreThanFourToolsAreEquallyClosest(self): """When more than four tools tie for closest, randomly select four.""" query = lookup_query.LookupQuery() query.tool_id = _TOOL_ID query.latitude = 0.0 query.longitude = 0.0 mock_fetched_tools = [ _createSliverTool( _TOOL_ID, site_id='abc01', latitude=1.0, longitude=1.0), _createSliverTool( _TOOL_ID, site_id='abc02', latitude=1.0, longitude=1.0), _createSliverTool( _TOOL_ID, site_id='abc03', latitude=1.0, longitude=1.0), _createSliverTool( _TOOL_ID, site_id='abc04', latitude=1.0, longitude=1.0), _createSliverTool( _TOOL_ID, site_id='abc05', latitude=1.0, longitude=1.0), _createSliverTool( _TOOL_ID, site_id='abc06', latitude=1.0, longitude=1.0), _createSliverTool( _TOOL_ID, site_id='cba01', latitude=5.0, longitude=5.0) ] # yapf: disable # The mock shuffle reverses the list, so we expect items 2...6 in # reverse order. query_results_expected = mock_fetched_tools[-2:-6:-1] # Make sure the resolver is fetching only online tools that match the # specified tool ID. tool_properties_expected = sliver_tool_fetcher.ToolProperties( tool_id=_TOOL_ID, status=message.STATUS_ONLINE) self.assertQueryResultWithRandomShuffle(query, mock_fetched_tools, query_results_expected, tool_properties_expected)
def testAnswerQueryWhenSingleToolIsClosest(self): """When a single tool is closest, return that tool.""" query = lookup_query.LookupQuery() query.tool_id = _TOOL_ID query.latitude = 0.0 query.longitude = 0.0 close_tool = _createSliverTool(_TOOL_ID, site_id='abc01', latitude=1.0, longitude=1.0) far_tool = _createSliverTool(_TOOL_ID, site_id='cba01', latitude=5.0, longitude=5.0) # Make sure the resolver is fetching only online tools that match the # specified tool ID. tool_properties_expected = sliver_tool_fetcher.ToolProperties( tool_id=_TOOL_ID, status=message.STATUS_ONLINE) mock_fetched_tools = [close_tool, far_tool] self.assertQueryResultSingleTool(query, mock_fetched_tools, close_tool, tool_properties_expected)
def testInitializeDefaultsToGeoPolicyWhenUserDefinedPolicyIsInvalidAndGeoDataIsAvailable( self): self.mock_query_params[message.POLICY] = 'invalid_policy' query = lookup_query.LookupQuery() query.initialize_from_http_request(self.mock_request) self.assertEqual(message.POLICY_GEO, query.policy)
def get(self): """Handles an HTTP GET request. The URL must be in the following format: 'http://mlab-ns.appspot.com/tool-name?query_string', where tool-name is one of the tools running on M-Lab. For more information about the URL and the supported arguments in the query string, see the design doc at http://goo.gl/48S22. """ query = lookup_query.LookupQuery() query.initialize_from_http_request(self.request) # Check right away whether we should proxy this request. url = reverse_proxy.try_reverse_proxy_url(query, datetime.datetime.now()) if url: # NB: if sending the proxy url is unsuccessful, then fall through to # regular request handling. success = self.send_proxy_response(url) if success: logging.info('[reverse_proxy],true,%s', url) return logging.info('Policy is %s', query.policy) client_signature = query.calculate_client_signature() lookup_resolver = resolver.new_resolver(query.policy, client_signature) sliver_tools = lookup_resolver.answer_query(query) if sliver_tools is None: # NOTE: at this point, we know that either the query is invalid # (e.g. bad tool_id) or that a valid query has no capacity. if model.is_valid_tool(query.tool_id): # A.K.A. "no capacity". return util.send_no_content(self) else: # Invalid tool, so report "404 Not Found". return util.send_not_found(self, query.response_format) if query.response_format == message.FORMAT_JSON: self.send_json_response(sliver_tools, query) elif query.response_format == message.FORMAT_HTML: self.send_html_response(sliver_tools, query) elif query.response_format == message.FORMAT_REDIRECT: self.send_redirect_response(sliver_tools, query) elif query.response_format == message.FORMAT_BT: self.send_bt_response(sliver_tools, query) elif query.response_format == message.FORMAT_MAP: candidates = lookup_resolver.get_candidates(query) self.send_map_response(sliver_tools, query, candidates) else: # TODO (claudiu) Discuss what should be the default behaviour. # I think json it's OK since is valid for all tools, while # redirect only applies to web-based tools. self.send_json_response(sliver_tools, query) # At this point, the client has received a response but the server has # not closed the connection. self.log_location(query, sliver_tools) # TODO (claudiu) Add a FORMAT_TYPE column in the BigQuery schema. self.log_request(query, sliver_tools)
def testInitializeAcceptsGeoPolicy(self): self.mock_query_params[message.POLICY] = message.POLICY_GEO query = lookup_query.LookupQuery() query.initialize_from_http_request(self.mock_request) self.assertEqual(message.POLICY_GEO, query.policy)
def testInitializeAcceptsValidUserDefinedFormat(self): self.mock_query_params[message.RESPONSE_FORMAT] = message.FORMAT_HTML query = lookup_query.LookupQuery() query.initialize_from_http_request(self.mock_request) self.assertEqual(query.response_format, message.FORMAT_HTML)
def testInitializeAcceptsValidToolAfv6(self): user_defined_af = message.ADDRESS_FAMILY_IPv6 self.mock_query_params[message.ADDRESS_FAMILY] = user_defined_af query = lookup_query.LookupQuery() query.initialize_from_http_request(self.mock_request) self.assertEqual(user_defined_af, query.tool_address_family)
def testInitializeIgnoresInvalidUserDefinedIp(self): """Ignore an invalid user-defined IP address.""" self.mock_query_params[message.REMOTE_ADDRESS] = 'invalid_ip' query = lookup_query.LookupQuery() query.initialize_from_http_request(self.mock_request) self.assertEqual(self.mock_request_ip, query.ip_address)