def fill_journeys(self, request_type, api_request, instance): krakens_call = get_kraken_calls(api_request) request = deepcopy(api_request) min_asked_journeys = get_or_default(request, 'min_nb_journeys', 1) min_journeys_calls = get_or_default(request, '_min_journeys_calls', 1) responses = [] nb_try = 0 while request is not None and \ ((nb_journeys(responses) < min_asked_journeys and nb_try < min_asked_journeys) or nb_try < min_journeys_calls): nb_try = nb_try + 1 tmp_resp = self.call_kraken(request_type, request, instance, krakens_call) _tag_by_mode(tmp_resp) _tag_direct_path(tmp_resp) journey_filter._filter_too_long_journeys(tmp_resp, request) responses.extend( tmp_resp) # we keep the error for building the response if nb_journeys(tmp_resp) == 0: # no new journeys found, we stop break request = self.create_next_kraken_request(request, tmp_resp) # we filter unwanted journeys by side effects journey_filter.filter_journeys(responses, instance, api_request) #We allow one more call to kraken if there is no valid journey. if nb_journeys(responses) == 0: min_journeys_calls = max(min_journeys_calls, 2) journey_filter.final_filter_journeys(responses, instance, api_request) pb_resp = merge_responses(responses) sort_journeys(pb_resp, instance.journey_order, api_request['clockwise']) compute_car_co2_emission(pb_resp, api_request, instance) tag_journeys(pb_resp) journey_filter.delete_journeys((pb_resp, ), api_request) type_journeys(pb_resp, api_request) culling_journeys(pb_resp, api_request) self._compute_pagination_links(pb_resp, instance, api_request['clockwise']) return pb_resp
def filter_journeys(responses, new_resp, instance, api_request): # we filter unwanted journeys in the new response # note that filter_journeys returns a generator which will be evaluated later filtered_new_resp = journey_filter.filter_journeys(new_resp, instance, api_request) # duplicate the generator tmp1, tmp2 = itertools.tee(filtered_new_resp) qualified_journeys = journey_filter.get_qualified_journeys(responses) # now we want to filter similar journeys in the new response which is done in 2 steps # In the first step, we compare journeys from the new response only , 2 by 2 # hopefully, it may lead to some early return for the second step to improve the perf a little # In the second step, we compare the journeys from the new response with those that have been qualified # already in the former iterations # note that the journey_pairs_pool is a list of 2-element tuple of journeys journey_pairs_pool = itertools.chain( # First step: compare journeys from the new response only itertools.combinations(tmp1, 2), # Second step: # we use the itertools.product to create combinations between qualified journeys and new journeys # Ex: # new_journeys = [n_1, n_2] # qualified_journeys = [q_1, q_2, q_3] # itertools.product(new_journeys, qualified_journeys) gives combinations as follows: # (n_1, q_1), (n_1, q_2),(n_1, q_3),(n_2, q_1),(n_2, q_2),(n_2, q_3) itertools.product(tmp2, qualified_journeys), ) journey_filter.filter_similar_vj_journeys(journey_pairs_pool, api_request)
def fill_journeys(self, request_type, api_request, instance): krakens_call = get_kraken_calls(api_request) request = deepcopy(api_request) min_asked_journeys = get_or_default(request, 'min_nb_journeys', 1) min_journeys_calls = get_or_default(request, '_min_journeys_calls', 1) responses = [] nb_try = 0 while request is not None and \ ((nb_journeys(responses) < min_asked_journeys and nb_try < min_asked_journeys) or nb_try < min_journeys_calls): nb_try = nb_try + 1 tmp_resp = self.call_kraken(request_type, request, instance, krakens_call) responses.extend( tmp_resp) # we keep the error for building the response if nb_journeys(tmp_resp) == 0: # no new journeys found, we stop break request = self.create_next_kraken_request(request, tmp_resp) # we filter unwanted journeys by side effects journey_filter.filter_journeys(responses, instance, api_request) journey_filter.final_filter_journeys(responses, instance, api_request) pb_resp = merge_responses(responses) sort_journeys(pb_resp, instance.journey_order, api_request['clockwise']) tag_journeys(pb_resp) type_journeys(pb_resp, api_request) culling_journeys(pb_resp, api_request) self._compute_pagination_links(pb_resp, instance) return pb_resp
def fill_journeys(self, request_type, api_request, instance): krakens_call = get_kraken_calls(api_request) request = deepcopy(api_request) min_asked_journeys = get_or_default(request, 'min_nb_journeys', 1) responses = [] last_nb_journeys = 0 while nb_journeys(responses) < min_asked_journeys: tmp_resp = self.call_kraken(request_type, request, instance, krakens_call) responses.extend(tmp_resp) new_nb_journeys = nb_journeys(responses) if new_nb_journeys == 0: #no new journeys found, we stop break #we filter unwanted journeys by side effects journey_filter.filter_journeys(responses, instance, request=request, original_request=api_request) if last_nb_journeys == new_nb_journeys: #we are stuck with the same number of journeys, we stops break last_nb_journeys = new_nb_journeys request = create_next_kraken_request(request, responses) pb_resp = merge_responses(responses) sort_journeys(pb_resp, instance.journey_order, request['clockwise']) tag_journeys(pb_resp) culling_journeys(pb_resp, request) return pb_resp
def fill_journeys(self, request_type, api_request, instance): krakens_call = get_kraken_calls(api_request) # sometimes we need to change the entrypoint id (eg if the id is from another autocomplete system) origin_detail = self.get_entrypoint_detail(api_request.get('origin'), instance) destination_detail = self.get_entrypoint_detail( api_request.get('destination'), instance) # we store the origin/destination detail in g to be able to use them after the marshall g.origin_detail = origin_detail g.destination_detail = destination_detail api_request['origin'] = get_kraken_id( origin_detail) or api_request.get('origin') api_request['destination'] = get_kraken_id( destination_detail) or api_request.get('destination') request = deepcopy(api_request) min_asked_journeys = get_or_default(request, 'min_nb_journeys', 1) min_journeys_calls = get_or_default(request, '_min_journeys_calls', 1) responses = [] nb_try = 0 while request is not None and \ ((nb_journeys(responses) < min_asked_journeys and nb_try < min_asked_journeys) or nb_try < min_journeys_calls): nb_try = nb_try + 1 tmp_resp = self.call_kraken(request_type, request, instance, krakens_call) _tag_by_mode(tmp_resp) _tag_direct_path(tmp_resp) _tag_bike_in_pt(tmp_resp) journey_filter._filter_too_long_journeys(tmp_resp, request) responses.extend( tmp_resp) # we keep the error for building the response if nb_journeys(tmp_resp) == 0: # no new journeys found, we stop break request = self.create_next_kraken_request(request, tmp_resp) # we filter unwanted journeys by side effects journey_filter.filter_journeys(responses, instance, api_request) #We allow one more call to kraken if there is no valid journey. if nb_journeys(responses) == 0: min_journeys_calls = max(min_journeys_calls, 2) journey_filter.final_filter_journeys(responses, instance, api_request) pb_resp = merge_responses(responses) sort_journeys(pb_resp, instance.journey_order, api_request['clockwise']) compute_car_co2_emission(pb_resp, api_request, instance) tag_journeys(pb_resp) journey_filter.delete_journeys((pb_resp, ), api_request) type_journeys(pb_resp, api_request) culling_journeys(pb_resp, api_request) self._compute_pagination_links(pb_resp, instance, api_request['clockwise']) return pb_resp
def fill_journeys(self, request_type, api_request, instance): logger = logging.getLogger(__name__) # sometimes we need to change the entrypoint id (eg if the id is from another autocomplete system) origin_detail = self.get_entrypoint_detail(api_request.get('origin'), instance) destination_detail = self.get_entrypoint_detail( api_request.get('destination'), instance) # we store the origin/destination detail in g to be able to use them after the marshall g.origin_detail = origin_detail g.destination_detail = destination_detail api_request['origin'] = get_kraken_id( origin_detail) or api_request.get('origin') api_request['destination'] = get_kraken_id( destination_detail) or api_request.get('destination') # building ridesharing request from "original" request ridesharing_req = deepcopy(api_request) if not api_request['origin_mode']: api_request['origin_mode'] = ['walking'] if not api_request['destination_mode']: api_request['destination_mode'] = ['walking'] krakens_call = get_kraken_calls(api_request) request = deepcopy(api_request) min_asked_journeys = get_or_default(request, 'min_nb_journeys', 1) min_journeys_calls = get_or_default(request, '_min_journeys_calls', 1) responses = [] nb_try = 0 while request is not None and \ ((nb_journeys(responses) < min_asked_journeys and nb_try < min_asked_journeys) or nb_try < min_journeys_calls): nb_try = nb_try + 1 tmp_resp = self.call_kraken(request_type, request, instance, krakens_call) _tag_by_mode(tmp_resp) _tag_direct_path(tmp_resp) _tag_bike_in_pt(tmp_resp) journey_filter._filter_too_long_journeys(tmp_resp, request) responses.extend( tmp_resp) # we keep the error for building the response if nb_journeys(tmp_resp) == 0: # no new journeys found, we stop break request = self.create_next_kraken_request(request, tmp_resp) # we filter unwanted journeys by side effects journey_filter.filter_journeys(responses, instance, api_request) #We allow one more call to kraken if there is no valid journey. if nb_journeys(responses) == 0: min_journeys_calls = max(min_journeys_calls, 2) journey_filter.final_filter_journeys(responses, instance, api_request) pb_resp = merge_responses(responses) sort_journeys(pb_resp, instance.journey_order, api_request['clockwise']) compute_car_co2_emission(pb_resp, api_request, instance) tag_journeys(pb_resp) if instance.ridesharing_services and \ ('ridesharing' in ridesharing_req['origin_mode'] or 'ridesharing' in ridesharing_req['destination_mode']): logging.getLogger(__name__).debug( 'trying to add ridesharing journeys') try: decorate_journeys(pb_resp, instance, api_request) except Exception: logger.exception('Error while retrieving ridesharing ads') else: for j in pb_resp.journeys: if 'ridesharing' in j.tags: journey_filter.mark_as_dead( j, 'no_matching_ridesharing_found') journey_filter.delete_journeys((pb_resp, ), api_request) type_journeys(pb_resp, api_request) culling_journeys(pb_resp, api_request) self._compute_pagination_links(pb_resp, instance, api_request['clockwise']) return pb_resp
def fill_journeys(self, request_type, api_request, instance): logger = logging.getLogger(__name__) if api_request['max_nb_journeys'] is not None and api_request[ 'max_nb_journeys'] <= 0: return response_pb2.Response() # sometimes we need to change the entrypoint id (eg if the id is from another autocomplete system) origin_detail = self.get_entrypoint_detail(api_request.get('origin'), instance) destination_detail = self.get_entrypoint_detail( api_request.get('destination'), instance) # we store the origin/destination detail in g to be able to use them after the marshall g.origin_detail = origin_detail g.destination_detail = destination_detail api_request['origin'] = get_kraken_id( origin_detail) or api_request.get('origin') api_request['destination'] = get_kraken_id( destination_detail) or api_request.get('destination') # building ridesharing request from "original" request ridesharing_req = deepcopy(api_request) if not api_request['origin_mode']: api_request['origin_mode'] = ['walking'] if not api_request['destination_mode']: api_request['destination_mode'] = ['walking'] # Return the possible couples combinations (origin_mode and destination_mode) krakens_call = get_kraken_calls(api_request) # We need the original request (api_request) for filtering, but request # is modified by create_next_kraken_request function. request = deepcopy(api_request) # min_nb_journeys option if request['min_nb_journeys']: min_nb_journeys = request['min_nb_journeys'] else: min_nb_journeys = 1 responses = [] nb_try = 0 nb_qualified_journeys = 0 nb_previously_qualified_journeys = 0 last_chance_retry = False min_journeys_calls = request.get('_min_journeys_calls', 1) max_journeys_calls = app.config.get('MAX_JOURNEYS_CALLS', 20) max_nb_calls = min(min_nb_journeys, max_journeys_calls) # Initialize a context for distributed distributed_context = self.get_context() while request is not None and ( (nb_qualified_journeys < min_nb_journeys and nb_try < max_nb_calls) or nb_try < min_journeys_calls): nb_try = nb_try + 1 # The parameter 'min_nb_journeys' isn't used in the following cases: # - If there's more than one single origin_mode and destination_mode couple. # - If there was no journey qualified in the previous response, the last chance request is limited if len(krakens_call) > 1 or last_chance_retry: request['min_nb_journeys'] = 0 elif api_request['min_nb_journeys']: min_nb_journeys_left = min_nb_journeys - nb_qualified_journeys request['min_nb_journeys'] = max(0, min_nb_journeys_left) new_resp = self.call_kraken(request_type, request, instance, krakens_call, distributed_context) _tag_by_mode(new_resp) _tag_direct_path(new_resp) _tag_bike_in_pt(new_resp) if nb_journeys(new_resp) == 0: # no new journeys found, we stop # we still append the new_resp because there are journeys that a tagged as dead probably responses.extend(new_resp) break request = self.create_next_kraken_request(request, new_resp) # we filter unwanted journeys in the new response # note that filter_journeys returns a generator which will be evaluated later filtered_new_resp = journey_filter.filter_journeys( new_resp, instance, api_request) # duplicate the generator tmp1, tmp2 = itertools.tee(filtered_new_resp) qualified_journeys = journey_filter.get_qualified_journeys( responses) # now we want to filter similar journeys in the new response which is done in 2 steps # In the first step, we compare journeys from the new response only , 2 by 2 # hopefully, it may lead to some early return for the second step to improve the perf a little # In the second step, we compare the journeys from the new response with those that have been qualified # already in the former iterations # note that the journey_pairs_pool is a list of 2-element tuple of journeys journey_pairs_pool = itertools.chain( # First step: compare journeys from the new response only itertools.combinations(tmp1, 2), # Second step: # we use the itertools.product to create combinations between qualified journeys and new journeys # Ex: # new_journeys = [n_1, n_2] # qualified_journeys = [q_1, q_2, q_3] # itertools.product(new_journeys, qualified_journeys) gives combinations as follows: # (n_1, q_1), (n_1, q_2),(n_1, q_3),(n_2, q_1),(n_2, q_2),(n_2, q_3) itertools.product(tmp2, qualified_journeys), ) journey_filter.filter_similar_vj_journeys(journey_pairs_pool, api_request) responses.extend( new_resp) # we keep the error for building the response nb_qualified_journeys = nb_journeys(responses) if api_request['timeframe_duration']: # If timeframe_duration is active, it is useless to recall Kraken, # it has already sent back what he could break if nb_previously_qualified_journeys == nb_qualified_journeys: # If there is no additional qualified journey in the kraken response, # another request is sent to try to find more journeys, just in case... if last_chance_retry: break last_chance_retry = True nb_previously_qualified_journeys = nb_qualified_journeys if nb_qualified_journeys == 0: min_journeys_calls = max(min_journeys_calls, 2) logger.debug('nb of call kraken: %i', nb_try) journey_filter.apply_final_journey_filters(responses, instance, api_request) pb_resp = merge_responses(responses, api_request['debug']) sort_journeys(pb_resp, instance.journey_order, api_request['clockwise']) compute_car_co2_emission(pb_resp, api_request, instance) tag_journeys(pb_resp) if instance.ridesharing_services and ( 'ridesharing' in ridesharing_req['origin_mode'] or 'ridesharing' in ridesharing_req['destination_mode']): logger.debug('trying to add ridesharing journeys') try: decorate_journeys(pb_resp, instance, api_request) except Exception: logger.exception('Error while retrieving ridesharing ads') else: for j in pb_resp.journeys: if 'ridesharing' in j.tags: journey_filter.mark_as_dead( j, api_request.get('debug'), 'no_matching_ridesharing_found') journey_filter.delete_journeys((pb_resp, ), api_request) type_journeys(pb_resp, api_request) culling_journeys(pb_resp, api_request) self._compute_pagination_links(pb_resp, instance, api_request['clockwise']) return pb_resp