def _call_here(self, url, params={}, http_method=requests.get, data={}, headers={}): self.log.debug('Here routing service, url: {}'.format(url)) try: r = self.breaker.call(http_method, url, timeout=self.timeout, params=params, data=data, headers=headers) self.record_call('ok') return r except pybreaker.CircuitBreakerError as e: self.log.error('Here routing service dead (error: {})'.format(e)) self.record_call('failure', reason='circuit breaker open') raise TechnicalError('HERE service not available') except requests.Timeout as t: self.log.error('Here routing service dead (error: {})'.format(t)) self.record_call('failure', reason='timeout') raise TechnicalError( 'impossible to access HERE service, timeout reached') except Exception as e: self.log.exception('Here routing error') self.record_call('failure', reason=str(e)) raise TechnicalError('impossible to access Here service')
def _check_response(cls, response, uri): if response is None: raise TechnicalError('impossible to access autocomplete service') if response.status_code == 404: raise UnknownObject(uri) if response.status_code != 200: raise TechnicalError('error in autocomplete request')
def get(self, request, instance): if not self.external_api: raise TechnicalError('global autocomplete not configured') params = self.make_params(request, instance) shape = request.get('shape', None) try: if shape: raw_response = requests.post(self.external_api, timeout=self.timeout, json=shape, params=params) else: raw_response = requests.get(self.external_api, timeout=self.timeout, params=params) except requests.Timeout: logging.getLogger(__name__).error('autocomplete request timeout') raise TechnicalError('external autocomplete service timeout') except: logging.getLogger(__name__).exception( 'error in autocomplete request') raise TechnicalError( 'impossible to access external autocomplete service') bragi_response = raw_response.json() from flask.ext.restful import marshal from jormungandr.interfaces.v1.Places import geocodejson return marshal(bragi_response, geocodejson)
def get_street_network_routing_matrix(self, origins, destinations, mode, max_duration, request, **kwargs): if len(origins) > 1: if len(destinations) > 1: logging.getLogger(__name__).error('routing matrix error, no unique center point') raise TechnicalError('routing matrix error, no unique center point') req = request_pb2.Request() req.requested_api = type_pb2.street_network_routing_matrix for o in origins: orig = req.sn_routing_matrix.origins.add() orig.place = 'coord:{c.lon}:{c.lat}'.format(c=get_pt_object_coord(o)) orig.access_duration = 0 for d in destinations: dest = req.sn_routing_matrix.destinations.add() dest.place = 'coord:{c.lon}:{c.lat}'.format(c=get_pt_object_coord(d)) dest.access_duration = 0 req.sn_routing_matrix.max_duration = max_duration req.sn_routing_matrix.mode = mode res = self._call_asgard(req) #TODO handle car park if res.HasField('error'): logging.getLogger(__name__).error('routing matrix query error {}'.format(res.error)) raise TechnicalError('routing matrix fail') return res.sn_routing_matrix
def call_bragi(url, method, **kwargs): try: return method(url, **kwargs) except requests.Timeout: logging.getLogger(__name__).error('autocomplete request timeout') raise TechnicalError('external autocomplete service timeout') except: logging.getLogger(__name__).exception('error in autocomplete request') raise TechnicalError('impossible to access external autocomplete service')
def _check_response(cls, response): if response == None: raise TechnicalError('impossible to access valhalla service') if response.status_code != 200: logging.getLogger(__name__).error( 'Valhalla service unavailable, impossible to query : {}' ' with response : {}'.format(response.url, response.text) ) raise TechnicalError('Valhalla service unavailable, impossible to query : {}'.format(response.url))
def make_url(self, end_point, uri=None): if end_point not in ['autocomplete', 'features', 'reverse']: raise TechnicalError('Unknown endpoint') if not self.host: raise TechnicalError('global autocomplete not configured') url = "{host}/{end_point}".format(host=self.host, end_point=end_point) if uri: url = '{url}/{uri}'.format(url=url, uri=uri) return url
def get_matrix_response(self, origins, destinations, post_resp): headers = {'Content-Type': 'application/json'} matrix_id = post_resp.get('matrixId', None) if matrix_id == None: raise TechnicalError( 'Here, invalid matrixId inside matrix POST response') # Here just expose a get to retreive matrix but you have to wait for the result to be available. # It is your own waiting loop ! get_url = self.matrix_service_url + '/' + str( matrix_id) + '?apiKey=' + str(self.apiKey) @retry(stop_max_delay=self.timeout * 1000, wait_fixed=self.lapse_time_matrix_to_retry * 1000) def _get_matrix_response(get_url): matrix_resp = requests.request("GET", get_url, headers=headers) if matrix_resp.status_code == 200: return self._create_matrix_response(matrix_resp.json(), origins, destinations) else: raise TechnicalError( 'Here, impossible to get matrix data. Should retry') return _get_matrix_response(get_url)
def get(self, id, region=None, lon=None, lat=None): args = self.parsers["get"].parse_args() args.update({ "uri": transform_id(id), "_current_datetime": datetime.utcnow() }) request_id = "places_{}".format(flask.request.id) args["request_id"] = request_id if any([region, lon, lat]): self.region = i_manager.get_region(region, lon, lat) timezone.set_request_timezone(self.region) response = i_manager.dispatch(args, "place_uri", instance_name=self.region) else: user = authentication.get_user(token=authentication.get_token(), abort_if_no_token=False) available_instances = get_all_available_instances(user) autocomplete = global_autocomplete.get('bragi') if not autocomplete: raise TechnicalError( 'world wide autocompletion service not available') response = autocomplete.get_by_uri(args["uri"], request_id=request_id, instances=available_instances) return response, 200
def get_street_network_routing_matrix(self, origins, destinations, mode, max_duration, request, **kwargs): speed_switcher = { "walking": request['walking_speed'], "bike": request['bike_speed'], "car": request['car_speed'], "bss": request['bss_speed'], } req = request_pb2.Request() req.requested_api = type_pb2.street_network_routing_matrix for o in origins: orig = req.sn_routing_matrix.origins.add() orig.place = 'coord:{c.lon}:{c.lat}'.format( c=get_pt_object_coord(o)) orig.access_duration = 0 for d in destinations: dest = req.sn_routing_matrix.destinations.add() dest.place = 'coord:{c.lon}:{c.lat}'.format( c=get_pt_object_coord(d)) dest.access_duration = 0 req.sn_routing_matrix.mode = mode req.sn_routing_matrix.speed = speed_switcher.get( mode, kwargs.get("walking")) req.sn_routing_matrix.max_duration = max_duration res = self._call_asgard(req) # TODO handle car park if res.HasField('error'): logging.getLogger(__name__).error( 'routing matrix query error {}'.format(res.error)) raise TechnicalError('routing matrix fail') return res.sn_routing_matrix
def get_autocomplete(self, requested_autocomplete): if not requested_autocomplete: return self.autocomplete autocomplete = global_autocomplete.get(requested_autocomplete) if not autocomplete: raise TechnicalError('autocomplete {} not available'.format(requested_autocomplete)) return autocomplete
def get(self, region=None, lon=None, lat=None): args = self.parsers["get"].parse_args() self._register_interpreted_parameters(args) if len(args['q']) == 0: abort(400, message="Search word absent") if args['disable_geojson']: g.disable_geojson = True user = authentication.get_user(token=authentication.get_token(), abort_if_no_token=False) args['shape'] = json.loads(user.shape) if user and user.shape else None # If a region or coords are asked, we do the search according # to the region, else, we do a word wide search if any([region, lon, lat]): self.region = i_manager.get_region(region, lon, lat) timezone.set_request_timezone(self.region) response = i_manager.dispatch(args, "places", instance_name=self.region) else: authentication.check_access_to_global_places(user) autocomplete = global_autocomplete.get('bragi') if autocomplete: response = autocomplete.get(args, instance=None) else: raise TechnicalError( 'world wide autocompletion service not available') return response, 200
def get(self, region=None, lon=None, lat=None): args = self.parsers["get"].parse_args() self._register_interpreted_parameters(args) if len(args['q']) == 0: abort(400, message="Search word absent") if args['disable_geojson']: g.disable_geojson = True user = authentication.get_user(token=authentication.get_token(), abort_if_no_token=False) if args['shape'] is None and user and user.shape: args['shape'] = json.loads(user.shape) if user and user.default_coord: if args['from'] is None: args['from'] = CoordFormat()(user.default_coord) else: if args['from'] == '': raise InvalidArguments("if 'from' is provided it cannot be null") # If a region or coords are asked, we do the search according # to the region, else, we do a word wide search if any([region, lon, lat]): self.region = i_manager.get_region(region, lon, lat) timezone.set_request_timezone(self.region) response = i_manager.dispatch(args, "places", instance_name=self.region) else: available_instances = get_all_available_instances(user) autocomplete = global_autocomplete.get('bragi') if not autocomplete: raise TechnicalError('world wide autocompletion service not available') response = autocomplete.get(args, instances=available_instances) return response, 200
def get_street_network_routing_matrix(self, instance, origins, destinations, street_network_mode, max_duration, request, request_id, **kwargs): # TODO: reverse is not handled as so far speed_switcher = jormungandr.street_network.utils.make_speed_switcher( request) # kraken can only manage 1-n request, so we reverse request if needed if len(origins) > 1: if len(destinations) > 1: logging.getLogger(__name__).error( 'routing matrix error, no unique center point') raise TechnicalError( 'routing matrix error, no unique center point') else: origins, destinations = destinations, origins req = self._create_sn_routing_matrix_request(origins, destinations, street_network_mode, max_duration, speed_switcher, **kwargs) res = instance.send_and_receive(req, request_id=request_id) self._check_for_error_and_raise(res) return res.sn_routing_matrix
def _get_matrix_response(get_url): matrix_resp = requests.request("GET", get_url, headers=headers) if matrix_resp.status_code == 200: return self._create_matrix_response(matrix_resp.json(), origins, destinations) else: raise TechnicalError( 'Here, impossible to get matrix data. Should retry')
def get_here_mode(mode): if mode == 'walking': return 'pedestrian' elif mode == 'bike': return 'bicycle' elif mode == 'car': return 'car' else: # HERE does not handle bss raise TechnicalError('HERE does not handle the mode {}'.format(mode))
def get(self, request, instance): if not self.external_api: raise TechnicalError('global autocomplete not configured') q = request['q'] url = '{endpoint}?q={q}'.format(endpoint=self.external_api, q=q) try: raw_response = requests.get(url, timeout=self.timeout) except requests.Timeout: logging.getLogger(__name__).error('autocomplete request timeout') raise TechnicalError('external autocomplete service timeout') except: logging.getLogger(__name__).exception( 'error in autocomplete request') raise TechnicalError( 'impossible to access external autocomplete service') return raw_response.json()
def get_street_network_routing_matrix(self, origins, destinations, street_network_mode, max_duration, request, **kwargs): # TODO: reverse is not handled as so far speed_switcher = { "walking": request['walking_speed'], "bike": request['bike_speed'], "car": request['car_speed'], "bss": request['bss_speed'], "ridesharing": request['car_no_park_speed'], } req = request_pb2.Request() req.requested_api = type_pb2.street_network_routing_matrix # kraken can only manage 1-n request, so we reverse request if needed if len(origins) > 1: if len(destinations) > 1: logging.getLogger(__name__).error( 'routing matrix error, no unique center point') raise TechnicalError( 'routing matrix error, no unique center point') else: origins, destinations = destinations, origins for o in origins: orig = req.sn_routing_matrix.origins.add() orig.place = get_uri_pt_object(o) orig.access_duration = 0 for d in destinations: dest = req.sn_routing_matrix.destinations.add() dest.place = get_uri_pt_object(d) dest.access_duration = 0 req.sn_routing_matrix.mode = street_network_mode req.sn_routing_matrix.speed = speed_switcher.get( street_network_mode, kwargs.get("walking")) req.sn_routing_matrix.max_duration = max_duration res = self.instance.send_and_receive(req) if res.HasField('error'): logging.getLogger(__name__).error( 'routing matrix query error {}'.format(res.error)) raise TechnicalError('routing matrix fail') return res.sn_routing_matrix
def get_timezone(): """ return the time zone associated with the query Note: for the moment ONLY ONE time zone is used for a region (a kraken instances) It is this default timezone that is returned in this method """ if not hasattr(g, 'timezone'): raise TechnicalError("No timezone set for this API") # the set_request_timezone has to be called at init return g.timezone
def get(self, region=None, lon=None, lat=None): args = self.parsers["get"].parse_args() self._register_interpreted_parameters(args) size_q = len(args['q']) if size_q == 0: abort(400, message="Search word absent") if size_q > 1024: abort( 413, message="Number of characters allowed for the search is 1024") if args['disable_geojson']: g.disable_geojson = True user = authentication.get_user(token=authentication.get_token(), abort_if_no_token=False) if args['shape'] is None and user and user.shape: args['shape'] = json.loads(user.shape) if not args.get("shape_scope[]") and user: args.update({"shape_scope[]": user.shape_scope}) if user and user.default_coord: if args['from'] is None: args['from'] = CoordFormat()(user.default_coord) else: if args['from'] == '': raise InvalidArguments( "if 'from' is provided it cannot be null") # If a region or coords are asked, we do the search according # to the region, else, we do a word wide search args["request_id"] = args.get('request_id', flask.request.id) if any([region, lon, lat]): self.region = i_manager.get_region(region, lon, lat) # when autocompletion is done on a specific coverage we want to filter on its shape if not args['shape']: instance = i_manager.instances.get(self.region) args['shape'] = build_instance_shape(instance) timezone.set_request_timezone(self.region) response = i_manager.dispatch(args, "places", instance_name=self.region) else: available_instances = get_all_available_instances(user) autocomplete = global_autocomplete.get('bragi') if not autocomplete: raise TechnicalError( 'world wide autocompletion service not available') response = autocomplete.get(args, instances=available_instances) return response, 200
def get_street_network_db(self, instance, streetnetwork_backend_conf): # type: (Instance, StreetNetworkBackend) -> Optional[AbstractStreetNetworkService] # Make sure we update the streetnetwork_backends list from the database before returning them self._update_config(instance) sn = self._streetnetwork_backends.get(streetnetwork_backend_conf, None) if sn is None: raise TechnicalError( 'impossible to find a streetnetwork module for instance {} with configuration {}' .format(instance, streetnetwork_backend_conf)) return sn
def get_street_network_routing_matrix(self, origins, destinations, mode, max_duration, request, **kwargs): params = self.get_matrix_params(origins, destinations, mode, max_duration, request) r = self._call_here(self.matrix_service_url, params=params) if r.status_code != 200: self.log.error('impossible to query HERE: {} with {} response: {}' .format(r.url, r.status_code, r.text)) raise TechnicalError('invalid HERE call, impossible to query: {}'.format(r.url)) resp_json = r.json() return self._get_matrix(resp_json, origins, destinations)
def get(self, id, region=None, lon=None, lat=None): args = self.parsers["get"].parse_args() args.update({ "uri": transform_id(id), "_current_datetime": datetime.utcnow() }) request_id = "places_{}".format(flask.request.id) args["request_id"] = request_id if any([region, lon, lat]): self.region = i_manager.get_region(region, lon, lat) timezone.set_request_timezone(self.region) response = i_manager.dispatch(args, "place_uri", instance_name=self.region) else: user = authentication.get_user(token=authentication.get_token(), abort_if_no_token=False) available_instances = get_all_available_instances( user, exclude_backend='kraken') # If no instance available most probably due to database error if (not user) and (not available_instances): raise TechnicalError( 'world wide autocompletion service not available temporarily' ) # If parameter '_autocomplete' is absent use 'bragi' as default value if args["_autocomplete"] is None: args["_autocomplete"] = app.config.get( 'DEFAULT_AUTOCOMPLETE_BACKEND', 'bragi') autocomplete = global_autocomplete.get(args["_autocomplete"]) if not autocomplete: raise TechnicalError( 'world wide autocompletion service not available') response = autocomplete.get_by_uri(args["uri"], request_id=request_id, instances=available_instances) return response, 200
def get_street_network(self, mode, request): overriden_sn_id = request.get('_street_network') if overriden_sn_id: def predicate(s): return s.sn_system_id == overriden_sn_id else: def predicate(s): return mode in s.modes sn = next((s for s in self.street_network_services if predicate(s)), None) if sn is None: raise TechnicalError('impossible to find a streetnetwork module for {} ({})'.format( mode, overriden_sn_id)) return sn
def get_here_mode(mode): map_mode = { fm.walking.name: 'pedestrian', fm.bike.name: 'bicycle', fm.car.name: 'car', fm.car_no_park.name: 'car', fm.taxi.name: 'car', fm.ridesharing.name: 'car', } try: return map_mode[mode] except KeyError: raise TechnicalError('HERE does not handle the mode {}'.format(mode))
def get_street_network_routing_matrix( self, instance, origins, destinations, mode, max_duration, request, **kwargs ): # for now valhalla only manages 1-n request, so we reverse request if needed if len(origins) > 1: if len(destinations) > 1: logging.getLogger(__name__).error('routing matrix error, no unique center point') raise TechnicalError('routing matrix error, no unique center point') data = self._make_request_arguments(mode, origins, destinations, request, api='sources_to_targets') r = self._call_valhalla('{}/{}'.format(self.service_url, 'sources_to_targets'), requests.post, data) self._check_response(r) resp_json = r.json() return self._get_matrix(resp_json, mode_park_cost=self.mode_park_cost.get(mode))
def _request(): with self.socket(self.instance.context) as socket: socket.send(request.SerializeToString()) # timeout is in second, we need it on millisecond if socket.poll(timeout=self.timeout * 1000) > 0: pb = socket.recv() resp = response_pb2.Response() resp.ParseFromString(pb) return resp else: socket.setsockopt(zmq.LINGER, 0) socket.close() self.logger.error('request on %s failed: %s', self.asgard_socket, six.text_type(request)) raise TechnicalError('asgard on {} failed'.format(self.asgard_socket))
def post_matrix_request(self, origins, destinations, request): post_data = self.matrix_payload(origins, destinations, request) params = {'apiKey': self.apiKey} headers = {'Content-Type': 'application/json'} post_resp = self._call_here(self.matrix_service_url, params=params, http_method=requests.post, data=post_data, headers=headers) post_resp.raise_for_status() if post_resp.status_code != 202: raise TechnicalError('Here, post return status code != 202') return post_resp.json()
def direct_path(self, mode, pt_object_origin, pt_object_destination, datetime, clockwise): url = self._format_url(mode, pt_object_origin, pt_object_destination) r = self._call_valhalla(url) if r == None: raise TechnicalError('impossible to access valhalla service') if r.status_code != 200: logging.getLogger(__name__).error( 'Valhalla service unavailable, impossible to query : {}'. format(r.url)) resp = response_pb2.Response() resp.status_code = r.status_code resp.error.message = 'Valhalla service unavailable, impossible to query : {}'.format( r.url) return resp resp_json = r.json() return self._get_response(resp_json, mode, pt_object_origin, pt_object_destination, datetime)
def __init__(self, context, name, zmq_socket, street_network_configurations, ridesharing_configurations, realtime_proxies_configuration, zmq_socket_type, autocomplete_type): self.geom = None self._sockets = queue.Queue() self.socket_path = zmq_socket self._scenario = None self._scenario_name = None self.lock = Lock() self.context = context self.name = name self.timezone = None # timezone will be fetched from the kraken self.publication_date = -1 self.is_initialized = False #kraken hasn't been called yet we don't have geom nor timezone self.breaker = pybreaker.CircuitBreaker(fail_max=app.config['CIRCUIT_BREAKER_MAX_INSTANCE_FAIL'], reset_timeout=app.config['CIRCUIT_BREAKER_INSTANCE_TIMEOUT_S']) self.georef = georef.Kraken(self) self.planner = planner.Kraken(self) street_network_configurations = _set_default_street_network_config(street_network_configurations) self.street_network_services = street_network.StreetNetwork.get_street_network_services(self, street_network_configurations) self.ridesharing_services = [] if ridesharing_configurations is not None: self.ridesharing_services = ridesharing_service.Ridesharing.\ get_ridesharing_services(self, ridesharing_configurations) self.ptref = ptref.PtRef(self) self.schedule = schedule.MixedSchedule(self) self.realtime_proxy_manager = realtime_schedule.RealtimeProxyManager(realtime_proxies_configuration, self) self.autocomplete = global_autocomplete.get(autocomplete_type) if not self.autocomplete: raise TechnicalError('impossible to find autocomplete system {} ' 'cannot initialize instance {}'.format(autocomplete_type, name)) self.zmq_socket_type = zmq_socket_type