def decorate_journeys(response, instance, request): # TODO: disable same journey schedule link for ridesharing journey? for journey in response.journeys: if 'ridesharing' not in journey.tags or to_be_deleted(journey): continue for i, section in enumerate(journey.sections): if section.street_network.mode == response_pb2.Ridesharing: section.additional_informations.append( response_pb2.HAS_DATETIME_ESTIMATED) period_extremity = None if len(journey.sections ) == 1: # direct path, we use the user input period_extremity = PeriodExtremity(request['datetime'], request['clockwise']) elif i == 0: # ridesharing on first section we want to arrive before the start of the pt period_extremity = PeriodExtremity(section.end_date_time, False) else: # ridesharing at the end, we search for solution starting after the end of the pt sections period_extremity = PeriodExtremity(section.begin_date_time, True) pb_rsjs, pb_tickets, pb_fps = build_ridesharing_journeys( section.origin, section.destination, period_extremity, instance) if not pb_rsjs: journey_filter.mark_as_dead( journey, 'no_matching_ridesharing_found') else: section.ridesharing_journeys.extend(pb_rsjs) response.tickets.extend(pb_tickets) response.feed_publishers.extend( (fp for fp in pb_fps if fp not in response.feed_publishers))
def decorate_journeys_with_ridesharing_offers(self, response, request, instance): greenlet_pool_actived = instance.greenlet_pool_for_ridesharing_services if greenlet_pool_actived: logging.info('ridesharing is called in async mode') else: logging.info('ridesharing is called in sequential mode') with FutureManager( instance.ridesharing_greenlet_pool_size) as future_manager: futures = {} for journey_idx, journey in enumerate(response.journeys): if 'ridesharing' not in journey.tags or to_be_deleted(journey): continue futures[journey_idx] = {} for section_idx, section in enumerate(journey.sections): if section.street_network.mode == response_pb2.Ridesharing: section.additional_informations.append( response_pb2.HAS_DATETIME_ESTIMATED) period_extremity = None if len(journey.sections ) == 1: # direct path, we use the user input period_extremity = PeriodExtremity( request['datetime'], request['clockwise']) elif ( section_idx == 0 ): # ridesharing on first section we want to arrive before the start of the pt period_extremity = PeriodExtremity( section.end_date_time, False) else: # ridesharing at the end, we search for solution starting after the end of the pt sections period_extremity = PeriodExtremity( section.begin_date_time, True) instance_params = self.InstanceParams.make_params( instance) if greenlet_pool_actived: futures[journey_idx][ section_idx] = future_manager.create_future( self.build_ridesharing_journeys, section.origin, section.destination, period_extremity, instance_params, ) else: pb_rsjs, pb_tickets, pb_fps = self.build_ridesharing_journeys( section.origin, section.destination, period_extremity, instance_params) self.add_new_ridesharing_results( pb_rsjs, pb_tickets, pb_fps, response, journey_idx, section_idx) if greenlet_pool_actived: for journey_idx in futures: for section_idx in futures[journey_idx]: pb_rsjs, pb_tickets, pb_fps = futures[journey_idx][ section_idx].wait_and_get() self.add_new_ridesharing_results( pb_rsjs, pb_tickets, pb_fps, response, journey_idx, section_idx)
def create_next_kraken_request(self, request, responses): """ modify the request to call the next (resp previous for non clockwise search) journeys in kraken to do that we find ask the next (resp previous) query datetime """ vjs = [j for r in responses for j in r.journeys if not journey_filter.to_be_deleted(j)] if request["clockwise"]: request['datetime'] = self.next_journey_datetime(vjs, request["clockwise"]) else: request['datetime'] = self.previous_journey_datetime(vjs, request["clockwise"]) if request['datetime'] is None: return None #TODO forbid ODTs return request
def culling_journeys(resp, request): """ Remove some journeys if there are too many of them to have max_nb_journeys journeys. resp.journeys should be sorted before this function is called The goal is to choose a bunch of journeys(max_nb_journeys) that covers as many as possible sections but have as few as possible sum(sections) Ex: From: Journey_1 : Line 1 -> Line 8 -> Bus 172 Journey_2 : Line 14 -> Line 6 -> Bus 165 Journey_3 : Line 14 -> Line 6 ->Line 8 -> Bus 165 Journey_4 : Line 1 -> Line 8 -> Bus 172 (this may happen when timeframe_duration or same_journey_schedule is used) We'd like to choose two journeys. The algo will return Journey_1 and Journey2. Note that Journey_4 is similar to the Journey_1 and will be ignored when max_nb_journeys<=3 Because With Journey_1 and Journey_3, they cover all lines but have 5 transfers in all With Journey_2 and Journey_3, they don't cover all lines(Line 1 is missing) and have 5 transfers in all With Journey_1 and Journey_2, they cover all lines and have only 4 transfers in all -> OK No removing done in debug """ logger = logging.getLogger(__name__) max_nb_journeys = request["max_nb_journeys"] if max_nb_journeys is None or max_nb_journeys >= len(resp.journeys): logger.debug('No need to cull journeys') return """ Why aggregating journeys before culling journeys? We have encountered severe slowness when combining max_nb_journeys(ex: 20) and a big timeframe_duration(ex: 86400s). It turned out that, with this configuration, kraken will return a lot of journeys(ex: 100 journeys) and the algorithm was trying to figure out the best solution over 5.35E+20 possible combinations ( 5.35E+20=Combination(100,20) )!! aggregated_journeys will group journeys that are similar('similar' is defined by 'Journeys that have the same sequence of sections are similar'), which reduces the number of possible combinations considerably """ aggregated_journeys, remaining_journeys = aggregate_journeys(resp.journeys) logger.debug( 'aggregated_journeys: {} remaining_journeys: {}'.format( len(aggregated_journeys), len(remaining_journeys) ) ) is_debug = request.get('debug') if max_nb_journeys >= len(aggregated_journeys): """ In this case, we return all aggregated_journeys plus earliest/latest journeys in remaining journeys """ for j in remaining_journeys[max(0, max_nb_journeys - len(aggregated_journeys)) :]: journey_filter.mark_as_dead( j, is_debug, 'max_nb_journeys >= len(aggregated_journeys), ' 'Filtered by max_nb_journeys' ) journey_filter.delete_journeys((resp,), request) return """ When max_nb_journeys < len(aggregated_journeys), we first remove all remaining journeys from final response because those journeys already have a similar journey in aggregated_journeys """ for j in remaining_journeys: journey_filter.mark_as_dead( j, is_debug, 'Filtered by max_nb_journeys, ' 'max_nb_journeys < len(aggregated_journeys)' ) logger.debug('Trying to culling the journeys') """ To create a candidates pool, we choose only journeys that are NOT tagged as 'comfort' and 'best' and we create a section set from that pool Ex: Journey_1 (Best): Line 14 -> Line 8 -> Bus 172 Journey_2 : Line 14 -> Line 6 -> Bus 165 Journey_3 : Line 14 -> Line 8 -> Bus 165 The candidate pool will be like [Journey_2, Journey_3] The sections set will be like set([Line 14, Line 6, Line 8, Bus 165]) """ candidates_pool, sections_set, idx_of_jrnys_must_keep = _build_candidate_pool_and_sections_set( aggregated_journeys ) nb_journeys_must_have = len(idx_of_jrnys_must_keep) logger.debug("There are {0} journeys we must keep".format(nb_journeys_must_have)) if max_nb_journeys <= nb_journeys_must_have: # At this point, max_nb_journeys is smaller than nb_journeys_must_have, we have to make choices def _inverse_selection(d, indexes): select = np.in1d(list(range(d.shape[0])), indexes) return d[~select] # Here we mark all journeys as dead that are not must-have for jrny in _inverse_selection(candidates_pool, idx_of_jrnys_must_keep): journey_filter.mark_as_dead(jrny, is_debug, 'Filtered by max_nb_journeys') if max_nb_journeys == nb_journeys_must_have: logger.debug('max_nb_journeys equals to nb_journeys_must_have') journey_filter.delete_journeys((resp,), request) return logger.debug( 'max_nb_journeys:{0} is smaller than nb_journeys_must_have:{1}'.format( request["max_nb_journeys"], nb_journeys_must_have ) ) # At this point, resp.journeys should contain only must-have journeys list_dict = collections.defaultdict(list) for jrny in resp.journeys: if not journey_filter.to_be_deleted(jrny): list_dict[jrny.type].append(jrny) sorted_by_type_journeys = [] for t in JOURNEY_TYPES_TO_RETAIN: sorted_by_type_journeys.extend(list_dict.get(t, [])) for jrny in sorted_by_type_journeys[max_nb_journeys:]: journey_filter.mark_as_dead(jrny, is_debug, 'Filtered by max_nb_journeys') journey_filter.delete_journeys((resp,), request) return logger.debug('Trying to find {0} journeys from {1}'.format(max_nb_journeys, candidates_pool.shape[0])) """ Ex: Journey_2 : Line 14 -> Line 6 -> Bus 165 Journey_3 : Line 14 -> Line 8 -> Bus 165 The candidate pool will be like [Journey_2, Journey_3] The sections set will be like set([Line 14, Line 6, Line 8, Bus 165]) selected_sections_matrix: [[1,1,0,1] -> journey_2 [1,0,1,1] -> journey_3 ] """ selected_sections_matrix = _build_selected_sections_matrix(sections_set, candidates_pool) best_indexes, selection_matrix = _get_sorted_solutions_indexes( selected_sections_matrix, max_nb_journeys, idx_of_jrnys_must_keep ) logger.debug("Nb best solutions: {0}".format(best_indexes.shape[0])) the_best_index = best_indexes[0] logger.debug("Trying to find the best of best") """ Let's find the best of best :) """ # If there're several solutions which have the same score of integrity and nb_sections if best_indexes.shape[0] != 1: requested_dt = request['datetime'] is_clockwise = request.get('clockwise', True) def combinations_sorter(v): # Hoping to find We sort the solution by the sum of journeys' pseudo duration return np.sum( ( get_pseudo_duration(jrny, requested_dt, is_clockwise) for jrny in np.array(candidates_pool)[np.where(selection_matrix[v, :])] ) ) the_best_index = min(best_indexes, key=combinations_sorter) logger.debug('Removing non selected journeys') for jrny in candidates_pool[np.where(selection_matrix[the_best_index, :] == 0)]: journey_filter.mark_as_dead(jrny, is_debug, 'Filtered by max_nb_journeys') journey_filter.delete_journeys((resp,), request)
def nb_journeys(responses): return sum(1 for r in responses for j in r.journeys if not journey_filter.to_be_deleted(j))
def culling_journeys(resp, request): """ Remove some journeys if there are too many of them to have max_nb_journeys journeys. resp.journeys should be sorted before this function is called The goal is to choose a bunch of journeys(max_nv_journeys) that covers as many as possible sections but have as few as possible sum(sections) Ex: From: Journey_1 : Line 1 -> Line 8 -> Bus 172 Journey_2 : Line 14 -> Line 6 -> Bus 165 Journey_3 : Line 14 -> Line 6 ->Line 8 -> Bus 165 W'd like to choose two journeys. The algo will return Journey_1 and Journey2. Because With Journey_1 and Journey_3, they cover all lines but have 5 transfers in all With Journey_2 and Journey_3, they don't cover all lines(Line 1 is missing) and have 5 transfers in all With Journey_1 and Journey_2, they cover all lines and have only 4 transfers in all -> OK No removing done in debug """ logger = logging.getLogger(__name__) if not request["max_nb_journeys"] or request["max_nb_journeys"] >= len( resp.journeys): logger.debug('No need to cull journeys') return logger.debug('Trying to culling the journeys') """ To create a candidates pool, we choose only journeys that are NOT tagged as 'comfort' and 'best' and we create a section set from that pool Ex: Journey_1 (Best): Line 14 -> Line 8 -> Bus 172 Journey_2 : Line 14 -> Line 6 -> Bus 165 Journey_3 : Line 14 -> Line 8 -> Bus 165 The candidate pool will be like [Journey_2, Journey_3] The sections set will be like set([Line 14, Line 6, Line 8, Bus 165]) """ candidates_pool, sections_set, idx_of_jrnys_must_keep = _build_candidate_pool_and_sections_set( resp) nb_journeys_must_have = len(idx_of_jrnys_must_keep) logger.debug( "There are {0} journeys we must keep".format(nb_journeys_must_have)) if (request["max_nb_journeys"] - nb_journeys_must_have) <= 0: # At this point, max_nb_journeys is smaller than nb_journeys_must_have, we have to make choices def _inverse_selection(d, indexes): select = np.in1d(list(range(d.shape[0])), indexes) return d[~select] # Here we mark all journeys as dead that are not must-have for jrny in _inverse_selection(candidates_pool, idx_of_jrnys_must_keep): journey_filter.mark_as_dead(jrny, 'Filtered by max_nb_journeys') if request["max_nb_journeys"] == nb_journeys_must_have: logger.debug('max_nb_journeys equals to nb_journeys_must_have') journey_filter.delete_journeys((resp, ), request) return logger.debug( 'max_nb_journeys:{0} is smaller than nb_journeys_must_have:{1}'. format(request["max_nb_journeys"], nb_journeys_must_have)) # At this point, resp.journeys should contain only must-have journeys list_dict = collections.defaultdict(list) for jrny in resp.journeys: if not journey_filter.to_be_deleted(jrny): list_dict[jrny.type].append(jrny) sorted_by_type_journeys = [] for t in JOURNEY_TYPES_TO_RETAIN: sorted_by_type_journeys.extend(list_dict.get(t, [])) for jrny in sorted_by_type_journeys[request["max_nb_journeys"]:]: journey_filter.mark_as_dead(jrny, 'Filtered by max_nb_journeys') journey_filter.delete_journeys((resp, ), request) return nb_journeys_to_find = request["max_nb_journeys"] logger.debug('Trying to find {0} journeys from {1}'.format( nb_journeys_to_find, candidates_pool.shape[0])) """ Ex: Journey_2 : Line 14 -> Line 6 -> Bus 165 Journey_3 : Line 14 -> Line 8 -> Bus 165 The candidate pool will be like [Journey_2, Journey_3] The sections set will be like set([Line 14, Line 6, Line 8, Bus 165]) selected_sections_matrix: [[1,1,0,1] -> journey_2 [1,0,1,1] -> journey_3 ] """ selected_sections_matrix = _build_selected_sections_matrix( sections_set, candidates_pool) best_indexes, selection_matrix = _get_sorted_solutions_indexes( selected_sections_matrix, nb_journeys_to_find, idx_of_jrnys_must_keep) logger.debug("Nb best solutions: {0}".format(best_indexes.shape[0])) the_best_index = best_indexes[0] logger.debug("Trying to find the best of best") """ Let's find the best of best :) """ # If there're several solutions which have the same score of integrity and nb_sections if best_indexes.shape[0] != 1: requested_dt = request['datetime'] is_clockwise = request.get('clockwise', True) def combinations_sorter(v): # Hoping to find We sort the solution by the sum of journeys' pseudo duration return np.sum((get_pseudo_duration(jrny, requested_dt, is_clockwise) for jrny in np.array(candidates_pool)[np.where( selection_matrix[v, :])])) the_best_index = min(best_indexes, key=combinations_sorter) logger.debug('Removing non selected journeys') for jrny in candidates_pool[np.where( selection_matrix[the_best_index, :] == 0)]: journey_filter.mark_as_dead(jrny, 'Filtered by max_nb_journeys') journey_filter.delete_journeys((resp, ), request)
def journeys_gen(list_responses): for r in list_responses: for j in r.journeys: if not to_be_deleted(j): yield j
def merge_responses(responses, debug): """ Merge all responses in one protobuf response """ merged_response = response_pb2.Response() for response_index, r in enumerate(responses): if r.HasField(str('error')) or not r.journeys: # we do not take responses with error, but if all responses have errors, we'll aggregate them continue change_ids(r, response_index) # we don't want to add a journey already there merged_response.journeys.extend(r.journeys) # we have to add the additional fares too # if at least one journey has the ticket we add it # and if the journey is not to be deleted in debug mode tickets_to_add = set( t for j in r.journeys if not (journey_filter.to_be_deleted(j) and not debug) for t in j.fare.ticket_id) merged_response.tickets.extend( (t for t in r.tickets if t.id in tickets_to_add)) initial_feed_publishers = {} for fp in merged_response.feed_publishers: initial_feed_publishers[fp.id] = fp # Add feed publishers from the qualified journeys only # Note : For BSS, it can happen that one journey in the response has returned a walking fallback. # If all other journeys in the response are to delete, the feed publisher will still be added # TODO: link feed publisher to a journey instead of a response with several journeys merged_response.feed_publishers.extend( fp for fp in r.feed_publishers if fp.id not in initial_feed_publishers and (debug or any( 'to_delete' not in j.tags for j in r.journeys))) # handle impacts for i in r.impacts: if any(other.uri == i.uri for other in merged_response.impacts): continue merged_response.impacts.extend([i]) if not merged_response.journeys: # we aggregate the errors found errors = { r.error.id: r.error for r in responses if r.HasField(str('error')) } # Only one errors field if len(errors) == 1: merged_response.error.id = list(errors.values())[0].id merged_response.error.message = list(errors.values())[0].message # we need to merge the errors elif len(errors) > 1: merged_response.error.id = response_pb2.Error.no_solution merged_response.error.message = "several errors occured: \n * {}".format( "\n * ".join([m.message for m in errors.values()])) return merged_response