def call_kraken(self, request_type, request, instance, krakens_call, context=None): """ For all krakens_call, call the kraken and aggregate the responses return the list of all responses """ # TODO: handle min_alternative_journeys # TODO: call first bss|bss and do not call walking|walking if no bss in first results record_custom_parameter('scenario', 'new_default') resp = [] logger = logging.getLogger(__name__) futures = [] reqctx = copy_flask_request_context() def worker(dep_mode, arr_mode, instance, request, flask_request_id): with copy_context_in_greenlet_stack(reqctx): return ( dep_mode, arr_mode, instance.send_and_receive( request, flask_request_id=flask_request_id), ) pool = gevent.pool.Pool(app.config.get('GREENLET_POOL_SIZE', 3)) for dep_mode, arr_mode, direct_path_type in krakens_call: pb_request = create_pb_request(request_type, request, dep_mode, arr_mode, direct_path_type) # we spawn a new greenlet, it won't have access to our thread local request object so we pass the request_id futures.append( pool.spawn(worker, dep_mode, arr_mode, instance, pb_request, flask_request_id=flask.request.id)) for future in gevent.iwait(futures): dep_mode, arr_mode, local_resp = future.get() # for log purpose we put and id in each journeys self.nb_kraken_calls += 1 for idx, j in enumerate(local_resp.journeys): j.internal_id = "{resp}-{j}".format(resp=self.nb_kraken_calls, j=idx) if dep_mode == 'ridesharing': switch_back_to_ridesharing(local_resp, True) if arr_mode == 'ridesharing': switch_back_to_ridesharing(local_resp, False) fill_uris(local_resp) resp.append(local_resp) logger.debug("for mode %s|%s we have found %s journeys", dep_mode, arr_mode, len(local_resp.journeys)) return resp
def call_kraken(self, request_type, request, instance, krakens_call, request_id, context): record_custom_parameter('scenario', 'distributed') logger = logging.getLogger(__name__) logger.warning("using experimental scenario!!") """ All spawned futures must be started(if they're not yet started) when leaving the scope. We do this to prevent the programme from being blocked in case where some un-started futures may hold threading locks. If we leave the scope without cleaning these futures, they may hold locks forever. Note that the cleaning process depends on the implementation of futures. """ try: with FutureManager(self.greenlet_pool_size) as future_manager, timed_logger( logger, 'call_kraken', request_id ): if request_type == type_pb2.ISOCHRONE: return self._scenario._compute_isochrone_common( future_manager, request, instance, krakens_call, type_pb2.ISOCHRONE ) elif request_type == type_pb2.PLANNER: return self._scenario._compute_journeys( future_manager, request, instance, krakens_call, context, type_pb2.PLANNER ) else: abort(400, message="This type of request is not supported with distributed") except PtException as e: logger.exception('') return [e.get()] except EntryPointException as e: logger.exception('') return [e.get()] except StreetNetworkException as e: return [e.get()]
def add_info_newrelic(response, *args, **kwargs): try: record_custom_parameter('navitia-request-id', request.id) token = get_token() user = get_user(token=token, abort_if_no_token=False) app_name = get_app_name(token) if user: record_custom_parameter('user_id', str(user.id)) record_custom_parameter('token_name', app_name) record_custom_parameter('version', __version__) coverages = get_used_coverages() if coverages: record_custom_parameter('coverage', coverages[0]) except: logger = logging.getLogger(__name__) logger.exception('error while reporting to newrelic:') return response
def call_kraken(self, request_type, request, instance, krakens_call, context): record_custom_parameter('scenario', 'distributed') logger = logging.getLogger(__name__) logger.warning("using experimental scenario!!") """ All spawned futures must be started(if they're not yet started) when leaving the scope. We do this to prevent the programme from being blocked in case where some un-started futures may hold threading locks. If we leave the scope without cleaning these futures, they may hold locks forever. Note that the cleaning process depends on the implementation of futures. """ try: with FutureManager() as future_manager: res = self._compute_all(future_manager, request, instance, krakens_call, context) return res except PtException as e: return [e.get()] except EntryPointException as e: return [e.get()]
def __init__(self): super(Scenario, self).__init__() self._scenario = Distributed() record_custom_parameter('scenario', 'distributed')
def call_kraken(self, req, instance, tag=None): record_custom_parameter('scenario', 'default') resp = None """ for all combinaison of departure and arrival mode we call kraken """ logger = logging.getLogger(__name__) futures = [] def worker(o_mode, d_mode, instance, request, request_id): return (o_mode, d_mode, instance.send_and_receive(request, request_id=request_id)) pool = gevent.pool.Pool(current_app.config.get('GREENLET_POOL_SIZE', 3)) for o_mode, d_mode in itertools.product(self.origin_modes, self.destination_modes): # since we use multiple green thread we have to copy the request local_req = copy.deepcopy(req) local_req.journeys.streetnetwork_params.origin_mode = o_mode local_req.journeys.streetnetwork_params.destination_mode = d_mode if o_mode == 'car' or (is_admin(req.journeys.origin[0].place) and is_admin( req.journeys.destination[0].place)): # we don't want direct path for car or for admin to admin journeys req.journeys.streetnetwork_params.enable_direct_path = False else: req.journeys.streetnetwork_params.enable_direct_path = True futures.append( pool.spawn(worker, o_mode, d_mode, instance, local_req, request_id=flask.request.id)) for future in gevent.iwait(futures): o_mode, d_mode, local_resp = future.get() if local_resp.response_type == response_pb2.ITINERARY_FOUND: # if a specific tag was provided, we tag the journeys # and we don't call the qualifier, it will be done after # with the journeys from the previous query if tag: for j in local_resp.journeys: j.type = tag else: # we qualify the journeys request_type = "arrival" if req.journeys.clockwise else "departure" qualifier_one(local_resp.journeys, request_type) fill_uris(local_resp) if not resp: resp = local_resp else: self.merge_response(resp, local_resp) if not resp: resp = local_resp logger.debug( "for mode %s|%s we have found %s journeys: %s", o_mode, d_mode, len(local_resp.journeys), [j.type for j in local_resp.journeys], ) return resp