def decorate_journeys(response, instance, request):
# TODO: disable same journey schedule link for ridesharing journey?
for journey in response.journeys:
if 'ridesharing' not in journey.tags or to_be_deleted(journey):
continue
for i, section in enumerate(journey.sections):
if section.street_network.mode == response_pb2.Ridesharing:
section.additional_informations.append(
response_pb2.HAS_DATETIME_ESTIMATED)
period_extremity = None
if len(journey.sections
) == 1: # direct path, we use the user input
period_extremity = PeriodExtremity(request['datetime'],
request['clockwise'])
elif i == 0: # ridesharing on first section we want to arrive before the start of the pt
period_extremity = PeriodExtremity(section.end_date_time,
False)
else: # ridesharing at the end, we search for solution starting after the end of the pt sections
period_extremity = PeriodExtremity(section.begin_date_time,
True)
pb_rsjs, pb_tickets, pb_fps = build_ridesharing_journeys(
section.origin, section.destination, period_extremity,
instance)
if not pb_rsjs:
journey_filter.mark_as_dead(
journey, 'no_matching_ridesharing_found')
else:
section.ridesharing_journeys.extend(pb_rsjs)
response.tickets.extend(pb_tickets)
response.feed_publishers.extend(
(fp for fp in pb_fps
if fp not in response.feed_publishers))
def decorate_journeys_with_ridesharing_offers(self, response, request,
instance):
greenlet_pool_actived = instance.greenlet_pool_for_ridesharing_services
if greenlet_pool_actived:
logging.info('ridesharing is called in async mode')
else:
logging.info('ridesharing is called in sequential mode')
with FutureManager(
instance.ridesharing_greenlet_pool_size) as future_manager:
futures = {}
for journey_idx, journey in enumerate(response.journeys):
if 'ridesharing' not in journey.tags or to_be_deleted(journey):
continue
futures[journey_idx] = {}
for section_idx, section in enumerate(journey.sections):
if section.street_network.mode == response_pb2.Ridesharing:
section.additional_informations.append(
response_pb2.HAS_DATETIME_ESTIMATED)
period_extremity = None
if len(journey.sections
) == 1: # direct path, we use the user input
period_extremity = PeriodExtremity(
request['datetime'], request['clockwise'])
elif (
section_idx == 0
): # ridesharing on first section we want to arrive before the start of the pt
period_extremity = PeriodExtremity(
section.end_date_time, False)
else: # ridesharing at the end, we search for solution starting after the end of the pt sections
period_extremity = PeriodExtremity(
section.begin_date_time, True)
instance_params = self.InstanceParams.make_params(
instance)
if greenlet_pool_actived:
futures[journey_idx][
section_idx] = future_manager.create_future(
self.build_ridesharing_journeys,
section.origin,
section.destination,
period_extremity,
instance_params,
)
else:
pb_rsjs, pb_tickets, pb_fps = self.build_ridesharing_journeys(
section.origin, section.destination,
period_extremity, instance_params)
self.add_new_ridesharing_results(
pb_rsjs, pb_tickets, pb_fps, response,
journey_idx, section_idx)
if greenlet_pool_actived:
for journey_idx in futures:
for section_idx in futures[journey_idx]:
pb_rsjs, pb_tickets, pb_fps = futures[journey_idx][
section_idx].wait_and_get()
self.add_new_ridesharing_results(
pb_rsjs, pb_tickets, pb_fps, response, journey_idx,
section_idx)
def create_next_kraken_request(self, request, responses):
"""
modify the request to call the next (resp previous for non clockwise search) journeys in kraken
to do that we find ask the next (resp previous) query datetime
"""
vjs = [j for r in responses for j in r.journeys if not journey_filter.to_be_deleted(j)]
if request["clockwise"]:
request['datetime'] = self.next_journey_datetime(vjs, request["clockwise"])
else:
request['datetime'] = self.previous_journey_datetime(vjs, request["clockwise"])
if request['datetime'] is None:
return None
#TODO forbid ODTs
return request
def culling_journeys(resp, request):
"""
Remove some journeys if there are too many of them to have max_nb_journeys journeys.
resp.journeys should be sorted before this function is called
The goal is to choose a bunch of journeys(max_nb_journeys) that covers as many as possible sections
but have as few as possible sum(sections)
Ex:
From:
Journey_1 : Line 1 -> Line 8 -> Bus 172
Journey_2 : Line 14 -> Line 6 -> Bus 165
Journey_3 : Line 14 -> Line 6 ->Line 8 -> Bus 165
Journey_4 : Line 1 -> Line 8 -> Bus 172 (this may happen when timeframe_duration or same_journey_schedule is used)
We'd like to choose two journeys. The algo will return Journey_1 and Journey2.
Note that Journey_4 is similar to the Journey_1 and will be ignored when max_nb_journeys<=3
Because
With Journey_1 and Journey_3, they cover all lines but have 5 transfers in all
With Journey_2 and Journey_3, they don't cover all lines(Line 1 is missing) and have 5 transfers in all
With Journey_1 and Journey_2, they cover all lines and have only 4 transfers in all -> OK
No removing done in debug
"""
logger = logging.getLogger(__name__)
max_nb_journeys = request["max_nb_journeys"]
if max_nb_journeys is None or max_nb_journeys >= len(resp.journeys):
logger.debug('No need to cull journeys')
return
"""
Why aggregating journeys before culling journeys?
We have encountered severe slowness when combining max_nb_journeys(ex: 20) and a big timeframe_duration(ex: 86400s).
It turned out that, with this configuration, kraken will return a lot of journeys(ex: 100 journeys) and the
algorithm was trying to figure out the best solution over 5.35E+20 possible combinations
( 5.35E+20=Combination(100,20) )!!
aggregated_journeys will group journeys that are similar('similar' is defined by 'Journeys that have the same sequence
of sections are similar'), which reduces the number of possible combinations considerably
"""
aggregated_journeys, remaining_journeys = aggregate_journeys(resp.journeys)
logger.debug(
'aggregated_journeys: {} remaining_journeys: {}'.format(
len(aggregated_journeys), len(remaining_journeys)
)
)
is_debug = request.get('debug')
if max_nb_journeys >= len(aggregated_journeys):
"""
In this case, we return all aggregated_journeys plus earliest/latest journeys in remaining journeys
"""
for j in remaining_journeys[max(0, max_nb_journeys - len(aggregated_journeys)) :]:
journey_filter.mark_as_dead(
j, is_debug, 'max_nb_journeys >= len(aggregated_journeys), ' 'Filtered by max_nb_journeys'
)
journey_filter.delete_journeys((resp,), request)
return
"""
When max_nb_journeys < len(aggregated_journeys), we first remove all remaining journeys from final response because
those journeys already have a similar journey in aggregated_journeys
"""
for j in remaining_journeys:
journey_filter.mark_as_dead(
j, is_debug, 'Filtered by max_nb_journeys, ' 'max_nb_journeys < len(aggregated_journeys)'
)
logger.debug('Trying to culling the journeys')
"""
To create a candidates pool, we choose only journeys that are NOT tagged as 'comfort' and 'best' and we create a
section set from that pool
Ex:
Journey_1 (Best): Line 14 -> Line 8 -> Bus 172
Journey_2 : Line 14 -> Line 6 -> Bus 165
Journey_3 : Line 14 -> Line 8 -> Bus 165
The candidate pool will be like [Journey_2, Journey_3]
The sections set will be like set([Line 14, Line 6, Line 8, Bus 165])
"""
candidates_pool, sections_set, idx_of_jrnys_must_keep = _build_candidate_pool_and_sections_set(
aggregated_journeys
)
nb_journeys_must_have = len(idx_of_jrnys_must_keep)
logger.debug("There are {0} journeys we must keep".format(nb_journeys_must_have))
if max_nb_journeys <= nb_journeys_must_have:
# At this point, max_nb_journeys is smaller than nb_journeys_must_have, we have to make choices
def _inverse_selection(d, indexes):
select = np.in1d(list(range(d.shape[0])), indexes)
return d[~select]
# Here we mark all journeys as dead that are not must-have
for jrny in _inverse_selection(candidates_pool, idx_of_jrnys_must_keep):
journey_filter.mark_as_dead(jrny, is_debug, 'Filtered by max_nb_journeys')
if max_nb_journeys == nb_journeys_must_have:
logger.debug('max_nb_journeys equals to nb_journeys_must_have')
journey_filter.delete_journeys((resp,), request)
return
logger.debug(
'max_nb_journeys:{0} is smaller than nb_journeys_must_have:{1}'.format(
request["max_nb_journeys"], nb_journeys_must_have
)
)
# At this point, resp.journeys should contain only must-have journeys
list_dict = collections.defaultdict(list)
for jrny in resp.journeys:
if not journey_filter.to_be_deleted(jrny):
list_dict[jrny.type].append(jrny)
sorted_by_type_journeys = []
for t in JOURNEY_TYPES_TO_RETAIN:
sorted_by_type_journeys.extend(list_dict.get(t, []))
for jrny in sorted_by_type_journeys[max_nb_journeys:]:
journey_filter.mark_as_dead(jrny, is_debug, 'Filtered by max_nb_journeys')
journey_filter.delete_journeys((resp,), request)
return
logger.debug('Trying to find {0} journeys from {1}'.format(max_nb_journeys, candidates_pool.shape[0]))
"""
Ex:
Journey_2 : Line 14 -> Line 6 -> Bus 165
Journey_3 : Line 14 -> Line 8 -> Bus 165
The candidate pool will be like [Journey_2, Journey_3]
The sections set will be like set([Line 14, Line 6, Line 8, Bus 165])
selected_sections_matrix:
[[1,1,0,1] -> journey_2
[1,0,1,1] -> journey_3
]
"""
selected_sections_matrix = _build_selected_sections_matrix(sections_set, candidates_pool)
best_indexes, selection_matrix = _get_sorted_solutions_indexes(
selected_sections_matrix, max_nb_journeys, idx_of_jrnys_must_keep
)
logger.debug("Nb best solutions: {0}".format(best_indexes.shape[0]))
the_best_index = best_indexes[0]
logger.debug("Trying to find the best of best")
"""
Let's find the best of best :)
"""
# If there're several solutions which have the same score of integrity and nb_sections
if best_indexes.shape[0] != 1:
requested_dt = request['datetime']
is_clockwise = request.get('clockwise', True)
def combinations_sorter(v):
# Hoping to find We sort the solution by the sum of journeys' pseudo duration
return np.sum(
(
get_pseudo_duration(jrny, requested_dt, is_clockwise)
for jrny in np.array(candidates_pool)[np.where(selection_matrix[v, :])]
)
)
the_best_index = min(best_indexes, key=combinations_sorter)
logger.debug('Removing non selected journeys')
for jrny in candidates_pool[np.where(selection_matrix[the_best_index, :] == 0)]:
journey_filter.mark_as_dead(jrny, is_debug, 'Filtered by max_nb_journeys')
journey_filter.delete_journeys((resp,), request)
def nb_journeys(responses):
return sum(1 for r in responses for j in r.journeys
if not journey_filter.to_be_deleted(j))
def culling_journeys(resp, request):
"""
Remove some journeys if there are too many of them to have max_nb_journeys journeys.
resp.journeys should be sorted before this function is called
The goal is to choose a bunch of journeys(max_nv_journeys) that covers as many as possible sections
but have as few as possible sum(sections)
Ex:
From:
Journey_1 : Line 1 -> Line 8 -> Bus 172
Journey_2 : Line 14 -> Line 6 -> Bus 165
Journey_3 : Line 14 -> Line 6 ->Line 8 -> Bus 165
W'd like to choose two journeys. The algo will return Journey_1 and Journey2.
Because
With Journey_1 and Journey_3, they cover all lines but have 5 transfers in all
With Journey_2 and Journey_3, they don't cover all lines(Line 1 is missing) and have 5 transfers in all
With Journey_1 and Journey_2, they cover all lines and have only 4 transfers in all -> OK
No removing done in debug
"""
logger = logging.getLogger(__name__)
if not request["max_nb_journeys"] or request["max_nb_journeys"] >= len(
resp.journeys):
logger.debug('No need to cull journeys')
return
logger.debug('Trying to culling the journeys')
"""
To create a candidates pool, we choose only journeys that are NOT tagged as 'comfort' and 'best' and we create a
section set from that pool
Ex:
Journey_1 (Best): Line 14 -> Line 8 -> Bus 172
Journey_2 : Line 14 -> Line 6 -> Bus 165
Journey_3 : Line 14 -> Line 8 -> Bus 165
The candidate pool will be like [Journey_2, Journey_3]
The sections set will be like set([Line 14, Line 6, Line 8, Bus 165])
"""
candidates_pool, sections_set, idx_of_jrnys_must_keep = _build_candidate_pool_and_sections_set(
resp)
nb_journeys_must_have = len(idx_of_jrnys_must_keep)
logger.debug(
"There are {0} journeys we must keep".format(nb_journeys_must_have))
if (request["max_nb_journeys"] - nb_journeys_must_have) <= 0:
# At this point, max_nb_journeys is smaller than nb_journeys_must_have, we have to make choices
def _inverse_selection(d, indexes):
select = np.in1d(list(range(d.shape[0])), indexes)
return d[~select]
# Here we mark all journeys as dead that are not must-have
for jrny in _inverse_selection(candidates_pool,
idx_of_jrnys_must_keep):
journey_filter.mark_as_dead(jrny, 'Filtered by max_nb_journeys')
if request["max_nb_journeys"] == nb_journeys_must_have:
logger.debug('max_nb_journeys equals to nb_journeys_must_have')
journey_filter.delete_journeys((resp, ), request)
return
logger.debug(
'max_nb_journeys:{0} is smaller than nb_journeys_must_have:{1}'.
format(request["max_nb_journeys"], nb_journeys_must_have))
# At this point, resp.journeys should contain only must-have journeys
list_dict = collections.defaultdict(list)
for jrny in resp.journeys:
if not journey_filter.to_be_deleted(jrny):
list_dict[jrny.type].append(jrny)
sorted_by_type_journeys = []
for t in JOURNEY_TYPES_TO_RETAIN:
sorted_by_type_journeys.extend(list_dict.get(t, []))
for jrny in sorted_by_type_journeys[request["max_nb_journeys"]:]:
journey_filter.mark_as_dead(jrny, 'Filtered by max_nb_journeys')
journey_filter.delete_journeys((resp, ), request)
return
nb_journeys_to_find = request["max_nb_journeys"]
logger.debug('Trying to find {0} journeys from {1}'.format(
nb_journeys_to_find, candidates_pool.shape[0]))
"""
Ex:
Journey_2 : Line 14 -> Line 6 -> Bus 165
Journey_3 : Line 14 -> Line 8 -> Bus 165
The candidate pool will be like [Journey_2, Journey_3]
The sections set will be like set([Line 14, Line 6, Line 8, Bus 165])
selected_sections_matrix:
[[1,1,0,1] -> journey_2
[1,0,1,1] -> journey_3
]
"""
selected_sections_matrix = _build_selected_sections_matrix(
sections_set, candidates_pool)
best_indexes, selection_matrix = _get_sorted_solutions_indexes(
selected_sections_matrix, nb_journeys_to_find, idx_of_jrnys_must_keep)
logger.debug("Nb best solutions: {0}".format(best_indexes.shape[0]))
the_best_index = best_indexes[0]
logger.debug("Trying to find the best of best")
"""
Let's find the best of best :)
"""
# If there're several solutions which have the same score of integrity and nb_sections
if best_indexes.shape[0] != 1:
requested_dt = request['datetime']
is_clockwise = request.get('clockwise', True)
def combinations_sorter(v):
# Hoping to find We sort the solution by the sum of journeys' pseudo duration
return np.sum((get_pseudo_duration(jrny, requested_dt,
is_clockwise)
for jrny in np.array(candidates_pool)[np.where(
selection_matrix[v, :])]))
the_best_index = min(best_indexes, key=combinations_sorter)
logger.debug('Removing non selected journeys')
for jrny in candidates_pool[np.where(
selection_matrix[the_best_index, :] == 0)]:
journey_filter.mark_as_dead(jrny, 'Filtered by max_nb_journeys')
journey_filter.delete_journeys((resp, ), request)
def journeys_gen(list_responses):
for r in list_responses:
for j in r.journeys:
if not to_be_deleted(j):
yield j
def merge_responses(responses, debug):
"""
Merge all responses in one protobuf response
"""
merged_response = response_pb2.Response()
for response_index, r in enumerate(responses):
if r.HasField(str('error')) or not r.journeys:
# we do not take responses with error, but if all responses have errors, we'll aggregate them
continue
change_ids(r, response_index)
# we don't want to add a journey already there
merged_response.journeys.extend(r.journeys)
# we have to add the additional fares too
# if at least one journey has the ticket we add it
# and if the journey is not to be deleted in debug mode
tickets_to_add = set(
t for j in r.journeys
if not (journey_filter.to_be_deleted(j) and not debug)
for t in j.fare.ticket_id)
merged_response.tickets.extend(
(t for t in r.tickets if t.id in tickets_to_add))
initial_feed_publishers = {}
for fp in merged_response.feed_publishers:
initial_feed_publishers[fp.id] = fp
# Add feed publishers from the qualified journeys only
# Note : For BSS, it can happen that one journey in the response has returned a walking fallback.
# If all other journeys in the response are to delete, the feed publisher will still be added
# TODO: link feed publisher to a journey instead of a response with several journeys
merged_response.feed_publishers.extend(
fp for fp in r.feed_publishers
if fp.id not in initial_feed_publishers and (debug or any(
'to_delete' not in j.tags for j in r.journeys)))
# handle impacts
for i in r.impacts:
if any(other.uri == i.uri for other in merged_response.impacts):
continue
merged_response.impacts.extend([i])
if not merged_response.journeys:
# we aggregate the errors found
errors = {
r.error.id: r.error
for r in responses if r.HasField(str('error'))
}
# Only one errors field
if len(errors) == 1:
merged_response.error.id = list(errors.values())[0].id
merged_response.error.message = list(errors.values())[0].message
# we need to merge the errors
elif len(errors) > 1:
merged_response.error.id = response_pb2.Error.no_solution
merged_response.error.message = "several errors occured: \n * {}".format(
"\n * ".join([m.message for m in errors.values()]))
return merged_response
