def filter_too_many_connections_test(): # yes, it's a hack... instance = lambda: None instance.max_additional_connections = 10 mocked_pb_response = build_mocked_response() mocked_request = {'debug': True, 'datetime': 1444903200, 'clockwise': True} journey_filter.apply_final_journey_filters([mocked_pb_response], instance, mocked_request) assert all('deleted_because_too_much_connections' not in j.tags for j in mocked_pb_response.journeys) instance.max_additional_connections = 0 mocked_pb_response = build_mocked_response() mocked_request = {'debug': True, 'datetime': 1444903200, 'clockwise': True} journey_filter.apply_final_journey_filters([mocked_pb_response], instance, mocked_request) assert sum('deleted_because_too_much_connections' not in j.tags for j in mocked_pb_response.journeys) == 17 # the best journey must be kept assert 'deleted_because_too_much_connections' not in mocked_pb_response.journeys[14].tags instance.max_additional_connections = 0 mocked_pb_response = build_mocked_response() mocked_request = {'_max_additional_connections': 1, 'debug': True, 'datetime': 1444903200, 'clockwise': True} journey_filter.apply_final_journey_filters([mocked_pb_response], instance, mocked_request) assert sum('deleted_because_too_much_connections' not in j.tags for j in mocked_pb_response.journeys) == 19 # the best journey must be kept assert 'deleted_because_too_much_connections' not in mocked_pb_response.journeys[14].tags
def filter_non_car_tagged_journey_test(): # yes, it's a hack... instance = lambda: None instance.max_additional_connections = 10 mocked_pb_response = build_mocked_car_response() mocked_request = {'debug': True, 'datetime': 1444903200, 'clockwise': True} def expected_deleted_non_car_journey(journeys, nb): assert sum('deleted_because_non_car_tagged_journey_filtered' in j.tags for j in journeys) == nb expected_deleted_non_car_journey(mocked_pb_response.journeys, 0) # Apply filter # We should see no filtered journey because we did not request for first_section_mode/origin_mode = ['car'] only journey_filter.apply_final_journey_filters([mocked_pb_response], instance, mocked_request) expected_deleted_non_car_journey(mocked_pb_response.journeys, 0) # Apply filter # with origin_mode = ['car', 'walking'] the filter does not apply to our journeys mocked_request['origin_mode'] = ['car', 'walking'] journey_filter.apply_final_journey_filters([mocked_pb_response], instance, mocked_request) expected_deleted_non_car_journey(mocked_pb_response.journeys, 0) # Apply filter # with origin_mode = ['car'] we expect to retain only 'car' tagged journey mocked_request['origin_mode'] = ['car'] journey_filter.apply_final_journey_filters([mocked_pb_response], instance, mocked_request) expected_deleted_non_car_journey(mocked_pb_response.journeys, 1)
def fill_journeys(self, request_type, api_request, instance): logger = logging.getLogger(__name__) if api_request['max_nb_journeys'] is not None and api_request['max_nb_journeys'] <= 0: return response_pb2.Response() # sometimes we need to change the entrypoint id (eg if the id is from another autocomplete system) origin_detail = self.get_entrypoint_detail(api_request.get('origin'), instance) destination_detail = self.get_entrypoint_detail(api_request.get('destination'), instance) # we store the origin/destination detail in g to be able to use them after the marshall g.origin_detail = origin_detail g.destination_detail = destination_detail api_request['origin'] = get_kraken_id(origin_detail) or api_request.get('origin') api_request['destination'] = get_kraken_id(destination_detail) or api_request.get('destination') # building ridesharing request from "original" request ridesharing_req = deepcopy(api_request) if not api_request['origin_mode']: api_request['origin_mode'] = ['walking'] if not api_request['destination_mode']: api_request['destination_mode'] = ['walking'] # Return the possible combinations (origin_mode ,destination_mode, direct_path_type) krakens_call = get_kraken_calls(api_request) # We need the original request (api_request) for filtering, but request # is modified by create_next_kraken_request function. request = deepcopy(api_request) # min_nb_journeys option if request['min_nb_journeys']: min_nb_journeys = request['min_nb_journeys'] else: min_nb_journeys = 1 responses = [] nb_try = 0 nb_qualified_journeys = 0 nb_previously_qualified_journeys = 0 last_chance_retry = False min_journeys_calls = request.get('_min_journeys_calls', 1) max_journeys_calls = app.config.get('MAX_JOURNEYS_CALLS', 20) max_nb_calls = min(min_nb_journeys, max_journeys_calls) # Initialize a context for distributed distributed_context = self.get_context() while request is not None and ( (nb_qualified_journeys < min_nb_journeys and nb_try < max_nb_calls) or nb_try < min_journeys_calls ): nb_try = nb_try + 1 # The parameter 'min_nb_journeys' isn't used in the following cases: # - If there's more than one single origin_mode and destination_mode couple. # - If there was no journey qualified in the previous response, the last chance request is limited if len(krakens_call) > 1 or last_chance_retry: request['min_nb_journeys'] = 0 elif api_request['min_nb_journeys']: min_nb_journeys_left = min_nb_journeys - nb_qualified_journeys request['min_nb_journeys'] = max(0, min_nb_journeys_left) new_resp = self.call_kraken(request_type, request, instance, krakens_call, distributed_context) _tag_by_mode(new_resp) _tag_direct_path(new_resp) _tag_bike_in_pt(new_resp) if journey_filter.nb_qualifed_journeys(new_resp) == 0: # no new journeys found, we stop # we still append the new_resp because there are journeys that a tagged as dead probably responses.extend(new_resp) break request = self.create_next_kraken_request(request, new_resp) filter_journeys(responses, new_resp, instance, api_request) responses.extend(new_resp) # we keep the error for building the response if api_request['timeframe_duration']: # If timeframe_duration is active, it is useless to recall Kraken, # it has already sent back what he could break nb_qualified_journeys = journey_filter.nb_qualifed_journeys(responses) if nb_previously_qualified_journeys == nb_qualified_journeys: # If there is no additional qualified journey in the kraken response, # another request is sent to try to find more journeys, just in case... if last_chance_retry: break last_chance_retry = True nb_previously_qualified_journeys = nb_qualified_journeys if nb_qualified_journeys == 0: min_journeys_calls = max(min_journeys_calls, 2) logger.debug('nb of call kraken: %i', nb_try) journey_filter.apply_final_journey_filters(responses, instance, api_request) self.finalise_journeys(api_request, responses, distributed_context, instance, api_request['debug']) pb_resp = merge_responses(responses, api_request['debug']) sort_journeys(pb_resp, instance.journey_order, api_request['clockwise']) compute_car_co2_emission(pb_resp, api_request, instance) tag_journeys(pb_resp) if instance.ridesharing_services and ( 'ridesharing' in ridesharing_req['origin_mode'] or 'ridesharing' in ridesharing_req['destination_mode'] ): logger.debug('trying to add ridesharing journeys') try: decorate_journeys(pb_resp, instance, api_request) except Exception: logger.exception('Error while retrieving ridesharing ads') else: for j in pb_resp.journeys: if 'ridesharing' in j.tags: journey_filter.mark_as_dead(j, api_request.get('debug'), 'no_matching_ridesharing_found') journey_filter.delete_journeys((pb_resp,), api_request) type_journeys(pb_resp, api_request) culling_journeys(pb_resp, api_request) self._compute_pagination_links(pb_resp, instance, api_request['clockwise']) return pb_resp
def fill_journeys(self, request_type, api_request, instance): logger = logging.getLogger(__name__) if api_request['max_nb_journeys'] is not None and api_request[ 'max_nb_journeys'] <= 0: return response_pb2.Response() # sometimes we need to change the entrypoint id (eg if the id is from another autocomplete system) origin_detail = self.get_entrypoint_detail(api_request.get('origin'), instance) destination_detail = self.get_entrypoint_detail( api_request.get('destination'), instance) # we store the origin/destination detail in g to be able to use them after the marshall g.origin_detail = origin_detail g.destination_detail = destination_detail api_request['origin'] = get_kraken_id( origin_detail) or api_request.get('origin') api_request['destination'] = get_kraken_id( destination_detail) or api_request.get('destination') # building ridesharing request from "original" request ridesharing_req = deepcopy(api_request) if not api_request['origin_mode']: api_request['origin_mode'] = ['walking'] if not api_request['destination_mode']: api_request['destination_mode'] = ['walking'] # Return the possible couples combinations (origin_mode and destination_mode) krakens_call = get_kraken_calls(api_request) # We need the original request (api_request) for filtering, but request # is modified by create_next_kraken_request function. request = deepcopy(api_request) # min_nb_journeys option if request['min_nb_journeys']: min_nb_journeys = request['min_nb_journeys'] else: min_nb_journeys = 1 responses = [] nb_try = 0 nb_qualified_journeys = 0 nb_previously_qualified_journeys = 0 last_chance_retry = False min_journeys_calls = request.get('_min_journeys_calls', 1) max_journeys_calls = app.config.get('MAX_JOURNEYS_CALLS', 20) max_nb_calls = min(min_nb_journeys, max_journeys_calls) # Initialize a context for distributed distributed_context = self.get_context() while request is not None and ( (nb_qualified_journeys < min_nb_journeys and nb_try < max_nb_calls) or nb_try < min_journeys_calls): nb_try = nb_try + 1 # The parameter 'min_nb_journeys' isn't used in the following cases: # - If there's more than one single origin_mode and destination_mode couple. # - If there was no journey qualified in the previous response, the last chance request is limited if len(krakens_call) > 1 or last_chance_retry: request['min_nb_journeys'] = 0 elif api_request['min_nb_journeys']: min_nb_journeys_left = min_nb_journeys - nb_qualified_journeys request['min_nb_journeys'] = max(0, min_nb_journeys_left) new_resp = self.call_kraken(request_type, request, instance, krakens_call, distributed_context) _tag_by_mode(new_resp) _tag_direct_path(new_resp) _tag_bike_in_pt(new_resp) if nb_journeys(new_resp) == 0: # no new journeys found, we stop # we still append the new_resp because there are journeys that a tagged as dead probably responses.extend(new_resp) break request = self.create_next_kraken_request(request, new_resp) # we filter unwanted journeys in the new response # note that filter_journeys returns a generator which will be evaluated later filtered_new_resp = journey_filter.filter_journeys( new_resp, instance, api_request) # duplicate the generator tmp1, tmp2 = itertools.tee(filtered_new_resp) qualified_journeys = journey_filter.get_qualified_journeys( responses) # now we want to filter similar journeys in the new response which is done in 2 steps # In the first step, we compare journeys from the new response only , 2 by 2 # hopefully, it may lead to some early return for the second step to improve the perf a little # In the second step, we compare the journeys from the new response with those that have been qualified # already in the former iterations # note that the journey_pairs_pool is a list of 2-element tuple of journeys journey_pairs_pool = itertools.chain( # First step: compare journeys from the new response only itertools.combinations(tmp1, 2), # Second step: # we use the itertools.product to create combinations between qualified journeys and new journeys # Ex: # new_journeys = [n_1, n_2] # qualified_journeys = [q_1, q_2, q_3] # itertools.product(new_journeys, qualified_journeys) gives combinations as follows: # (n_1, q_1), (n_1, q_2),(n_1, q_3),(n_2, q_1),(n_2, q_2),(n_2, q_3) itertools.product(tmp2, qualified_journeys), ) journey_filter.filter_similar_vj_journeys(journey_pairs_pool, api_request) responses.extend( new_resp) # we keep the error for building the response nb_qualified_journeys = nb_journeys(responses) if api_request['timeframe_duration']: # If timeframe_duration is active, it is useless to recall Kraken, # it has already sent back what he could break if nb_previously_qualified_journeys == nb_qualified_journeys: # If there is no additional qualified journey in the kraken response, # another request is sent to try to find more journeys, just in case... if last_chance_retry: break last_chance_retry = True nb_previously_qualified_journeys = nb_qualified_journeys if nb_qualified_journeys == 0: min_journeys_calls = max(min_journeys_calls, 2) logger.debug('nb of call kraken: %i', nb_try) journey_filter.apply_final_journey_filters(responses, instance, api_request) pb_resp = merge_responses(responses, api_request['debug']) sort_journeys(pb_resp, instance.journey_order, api_request['clockwise']) compute_car_co2_emission(pb_resp, api_request, instance) tag_journeys(pb_resp) if instance.ridesharing_services and ( 'ridesharing' in ridesharing_req['origin_mode'] or 'ridesharing' in ridesharing_req['destination_mode']): logger.debug('trying to add ridesharing journeys') try: decorate_journeys(pb_resp, instance, api_request) except Exception: logger.exception('Error while retrieving ridesharing ads') else: for j in pb_resp.journeys: if 'ridesharing' in j.tags: journey_filter.mark_as_dead( j, api_request.get('debug'), 'no_matching_ridesharing_found') journey_filter.delete_journeys((pb_resp, ), api_request) type_journeys(pb_resp, api_request) culling_journeys(pb_resp, api_request) self._compute_pagination_links(pb_resp, instance, api_request['clockwise']) return pb_resp