def _call_here(self,
url,
params={},
http_method=requests.get,
data={},
headers={}):
self.log.debug('Here routing service, url: {}'.format(url))
try:
r = self.breaker.call(http_method,
url,
timeout=self.timeout,
params=params,
data=data,
headers=headers)
self.record_call('ok')
return r
except pybreaker.CircuitBreakerError as e:
self.log.error('Here routing service dead (error: {})'.format(e))
self.record_call('failure', reason='circuit breaker open')
raise TechnicalError('HERE service not available')
except requests.Timeout as t:
self.log.error('Here routing service dead (error: {})'.format(t))
self.record_call('failure', reason='timeout')
raise TechnicalError(
'impossible to access HERE service, timeout reached')
except Exception as e:
self.log.exception('Here routing error')
self.record_call('failure', reason=str(e))
raise TechnicalError('impossible to access Here service')
def _check_response(cls, response, uri):
if response is None:
raise TechnicalError('impossible to access autocomplete service')
if response.status_code == 404:
raise UnknownObject(uri)
if response.status_code != 200:
raise TechnicalError('error in autocomplete request')
def get(self, request, instance):
if not self.external_api:
raise TechnicalError('global autocomplete not configured')
params = self.make_params(request, instance)
shape = request.get('shape', None)
try:
if shape:
raw_response = requests.post(self.external_api,
timeout=self.timeout,
json=shape,
params=params)
else:
raw_response = requests.get(self.external_api,
timeout=self.timeout,
params=params)
except requests.Timeout:
logging.getLogger(__name__).error('autocomplete request timeout')
raise TechnicalError('external autocomplete service timeout')
except:
logging.getLogger(__name__).exception(
'error in autocomplete request')
raise TechnicalError(
'impossible to access external autocomplete service')
bragi_response = raw_response.json()
from flask.ext.restful import marshal
from jormungandr.interfaces.v1.Places import geocodejson
return marshal(bragi_response, geocodejson)
def get_street_network_routing_matrix(self, origins, destinations, mode, max_duration, request, **kwargs):
if len(origins) > 1:
if len(destinations) > 1:
logging.getLogger(__name__).error('routing matrix error, no unique center point')
raise TechnicalError('routing matrix error, no unique center point')
req = request_pb2.Request()
req.requested_api = type_pb2.street_network_routing_matrix
for o in origins:
orig = req.sn_routing_matrix.origins.add()
orig.place = 'coord:{c.lon}:{c.lat}'.format(c=get_pt_object_coord(o))
orig.access_duration = 0
for d in destinations:
dest = req.sn_routing_matrix.destinations.add()
dest.place = 'coord:{c.lon}:{c.lat}'.format(c=get_pt_object_coord(d))
dest.access_duration = 0
req.sn_routing_matrix.max_duration = max_duration
req.sn_routing_matrix.mode = mode
res = self._call_asgard(req)
#TODO handle car park
if res.HasField('error'):
logging.getLogger(__name__).error('routing matrix query error {}'.format(res.error))
raise TechnicalError('routing matrix fail')
return res.sn_routing_matrix
def call_bragi(url, method, **kwargs):
try:
return method(url, **kwargs)
except requests.Timeout:
logging.getLogger(__name__).error('autocomplete request timeout')
raise TechnicalError('external autocomplete service timeout')
except:
logging.getLogger(__name__).exception('error in autocomplete request')
raise TechnicalError('impossible to access external autocomplete service')
def _check_response(cls, response):
if response == None:
raise TechnicalError('impossible to access valhalla service')
if response.status_code != 200:
logging.getLogger(__name__).error(
'Valhalla service unavailable, impossible to query : {}'
' with response : {}'.format(response.url, response.text)
)
raise TechnicalError('Valhalla service unavailable, impossible to query : {}'.format(response.url))
def make_url(self, end_point, uri=None):
if end_point not in ['autocomplete', 'features', 'reverse']:
raise TechnicalError('Unknown endpoint')
if not self.host:
raise TechnicalError('global autocomplete not configured')
url = "{host}/{end_point}".format(host=self.host, end_point=end_point)
if uri:
url = '{url}/{uri}'.format(url=url, uri=uri)
return url
def get_matrix_response(self, origins, destinations, post_resp):
headers = {'Content-Type': 'application/json'}
matrix_id = post_resp.get('matrixId', None)
if matrix_id == None:
raise TechnicalError(
'Here, invalid matrixId inside matrix POST response')
# Here just expose a get to retreive matrix but you have to wait for the result to be available.
# It is your own waiting loop !
get_url = self.matrix_service_url + '/' + str(
matrix_id) + '?apiKey=' + str(self.apiKey)
@retry(stop_max_delay=self.timeout * 1000,
wait_fixed=self.lapse_time_matrix_to_retry * 1000)
def _get_matrix_response(get_url):
matrix_resp = requests.request("GET", get_url, headers=headers)
if matrix_resp.status_code == 200:
return self._create_matrix_response(matrix_resp.json(),
origins, destinations)
else:
raise TechnicalError(
'Here, impossible to get matrix data. Should retry')
return _get_matrix_response(get_url)
def get(self, id, region=None, lon=None, lat=None):
args = self.parsers["get"].parse_args()
args.update({
"uri": transform_id(id),
"_current_datetime": datetime.utcnow()
})
request_id = "places_{}".format(flask.request.id)
args["request_id"] = request_id
if any([region, lon, lat]):
self.region = i_manager.get_region(region, lon, lat)
timezone.set_request_timezone(self.region)
response = i_manager.dispatch(args,
"place_uri",
instance_name=self.region)
else:
user = authentication.get_user(token=authentication.get_token(),
abort_if_no_token=False)
available_instances = get_all_available_instances(user)
autocomplete = global_autocomplete.get('bragi')
if not autocomplete:
raise TechnicalError(
'world wide autocompletion service not available')
response = autocomplete.get_by_uri(args["uri"],
request_id=request_id,
instances=available_instances)
return response, 200
def get_street_network_routing_matrix(self, origins, destinations, mode,
max_duration, request, **kwargs):
speed_switcher = {
"walking": request['walking_speed'],
"bike": request['bike_speed'],
"car": request['car_speed'],
"bss": request['bss_speed'],
}
req = request_pb2.Request()
req.requested_api = type_pb2.street_network_routing_matrix
for o in origins:
orig = req.sn_routing_matrix.origins.add()
orig.place = 'coord:{c.lon}:{c.lat}'.format(
c=get_pt_object_coord(o))
orig.access_duration = 0
for d in destinations:
dest = req.sn_routing_matrix.destinations.add()
dest.place = 'coord:{c.lon}:{c.lat}'.format(
c=get_pt_object_coord(d))
dest.access_duration = 0
req.sn_routing_matrix.mode = mode
req.sn_routing_matrix.speed = speed_switcher.get(
mode, kwargs.get("walking"))
req.sn_routing_matrix.max_duration = max_duration
res = self._call_asgard(req)
# TODO handle car park
if res.HasField('error'):
logging.getLogger(__name__).error(
'routing matrix query error {}'.format(res.error))
raise TechnicalError('routing matrix fail')
return res.sn_routing_matrix
def get_autocomplete(self, requested_autocomplete):
if not requested_autocomplete:
return self.autocomplete
autocomplete = global_autocomplete.get(requested_autocomplete)
if not autocomplete:
raise TechnicalError('autocomplete {} not available'.format(requested_autocomplete))
return autocomplete
def get(self, region=None, lon=None, lat=None):
args = self.parsers["get"].parse_args()
self._register_interpreted_parameters(args)
if len(args['q']) == 0:
abort(400, message="Search word absent")
if args['disable_geojson']:
g.disable_geojson = True
user = authentication.get_user(token=authentication.get_token(),
abort_if_no_token=False)
args['shape'] = json.loads(user.shape) if user and user.shape else None
# If a region or coords are asked, we do the search according
# to the region, else, we do a word wide search
if any([region, lon, lat]):
self.region = i_manager.get_region(region, lon, lat)
timezone.set_request_timezone(self.region)
response = i_manager.dispatch(args,
"places",
instance_name=self.region)
else:
authentication.check_access_to_global_places(user)
autocomplete = global_autocomplete.get('bragi')
if autocomplete:
response = autocomplete.get(args, instance=None)
else:
raise TechnicalError(
'world wide autocompletion service not available')
return response, 200
def get(self, region=None, lon=None, lat=None):
args = self.parsers["get"].parse_args()
self._register_interpreted_parameters(args)
if len(args['q']) == 0:
abort(400, message="Search word absent")
if args['disable_geojson']:
g.disable_geojson = True
user = authentication.get_user(token=authentication.get_token(), abort_if_no_token=False)
if args['shape'] is None and user and user.shape:
args['shape'] = json.loads(user.shape)
if user and user.default_coord:
if args['from'] is None:
args['from'] = CoordFormat()(user.default_coord)
else:
if args['from'] == '':
raise InvalidArguments("if 'from' is provided it cannot be null")
# If a region or coords are asked, we do the search according
# to the region, else, we do a word wide search
if any([region, lon, lat]):
self.region = i_manager.get_region(region, lon, lat)
timezone.set_request_timezone(self.region)
response = i_manager.dispatch(args, "places", instance_name=self.region)
else:
available_instances = get_all_available_instances(user)
autocomplete = global_autocomplete.get('bragi')
if not autocomplete:
raise TechnicalError('world wide autocompletion service not available')
response = autocomplete.get(args, instances=available_instances)
return response, 200
def get_street_network_routing_matrix(self, instance, origins,
destinations, street_network_mode,
max_duration, request, request_id,
**kwargs):
# TODO: reverse is not handled as so far
speed_switcher = jormungandr.street_network.utils.make_speed_switcher(
request)
# kraken can only manage 1-n request, so we reverse request if needed
if len(origins) > 1:
if len(destinations) > 1:
logging.getLogger(__name__).error(
'routing matrix error, no unique center point')
raise TechnicalError(
'routing matrix error, no unique center point')
else:
origins, destinations = destinations, origins
req = self._create_sn_routing_matrix_request(origins, destinations,
street_network_mode,
max_duration,
speed_switcher, **kwargs)
res = instance.send_and_receive(req, request_id=request_id)
self._check_for_error_and_raise(res)
return res.sn_routing_matrix
def _get_matrix_response(get_url):
matrix_resp = requests.request("GET", get_url, headers=headers)
if matrix_resp.status_code == 200:
return self._create_matrix_response(matrix_resp.json(),
origins, destinations)
else:
raise TechnicalError(
'Here, impossible to get matrix data. Should retry')
def get_here_mode(mode):
if mode == 'walking':
return 'pedestrian'
elif mode == 'bike':
return 'bicycle'
elif mode == 'car':
return 'car'
else: # HERE does not handle bss
raise TechnicalError('HERE does not handle the mode {}'.format(mode))
def get(self, request, instance):
if not self.external_api:
raise TechnicalError('global autocomplete not configured')
q = request['q']
url = '{endpoint}?q={q}'.format(endpoint=self.external_api, q=q)
try:
raw_response = requests.get(url, timeout=self.timeout)
except requests.Timeout:
logging.getLogger(__name__).error('autocomplete request timeout')
raise TechnicalError('external autocomplete service timeout')
except:
logging.getLogger(__name__).exception(
'error in autocomplete request')
raise TechnicalError(
'impossible to access external autocomplete service')
return raw_response.json()
def get_street_network_routing_matrix(self, origins, destinations,
street_network_mode, max_duration,
request, **kwargs):
# TODO: reverse is not handled as so far
speed_switcher = {
"walking": request['walking_speed'],
"bike": request['bike_speed'],
"car": request['car_speed'],
"bss": request['bss_speed'],
"ridesharing": request['car_no_park_speed'],
}
req = request_pb2.Request()
req.requested_api = type_pb2.street_network_routing_matrix
# kraken can only manage 1-n request, so we reverse request if needed
if len(origins) > 1:
if len(destinations) > 1:
logging.getLogger(__name__).error(
'routing matrix error, no unique center point')
raise TechnicalError(
'routing matrix error, no unique center point')
else:
origins, destinations = destinations, origins
for o in origins:
orig = req.sn_routing_matrix.origins.add()
orig.place = get_uri_pt_object(o)
orig.access_duration = 0
for d in destinations:
dest = req.sn_routing_matrix.destinations.add()
dest.place = get_uri_pt_object(d)
dest.access_duration = 0
req.sn_routing_matrix.mode = street_network_mode
req.sn_routing_matrix.speed = speed_switcher.get(
street_network_mode, kwargs.get("walking"))
req.sn_routing_matrix.max_duration = max_duration
res = self.instance.send_and_receive(req)
if res.HasField('error'):
logging.getLogger(__name__).error(
'routing matrix query error {}'.format(res.error))
raise TechnicalError('routing matrix fail')
return res.sn_routing_matrix
def get_timezone():
"""
return the time zone associated with the query
Note: for the moment ONLY ONE time zone is used for a region (a kraken instances)
It is this default timezone that is returned in this method
"""
if not hasattr(g, 'timezone'):
raise TechnicalError("No timezone set for this API") # the set_request_timezone has to be called at init
return g.timezone
def get(self, region=None, lon=None, lat=None):
args = self.parsers["get"].parse_args()
self._register_interpreted_parameters(args)
size_q = len(args['q'])
if size_q == 0:
abort(400, message="Search word absent")
if size_q > 1024:
abort(
413,
message="Number of characters allowed for the search is 1024")
if args['disable_geojson']:
g.disable_geojson = True
user = authentication.get_user(token=authentication.get_token(),
abort_if_no_token=False)
if args['shape'] is None and user and user.shape:
args['shape'] = json.loads(user.shape)
if not args.get("shape_scope[]") and user:
args.update({"shape_scope[]": user.shape_scope})
if user and user.default_coord:
if args['from'] is None:
args['from'] = CoordFormat()(user.default_coord)
else:
if args['from'] == '':
raise InvalidArguments(
"if 'from' is provided it cannot be null")
# If a region or coords are asked, we do the search according
# to the region, else, we do a word wide search
args["request_id"] = args.get('request_id', flask.request.id)
if any([region, lon, lat]):
self.region = i_manager.get_region(region, lon, lat)
# when autocompletion is done on a specific coverage we want to filter on its shape
if not args['shape']:
instance = i_manager.instances.get(self.region)
args['shape'] = build_instance_shape(instance)
timezone.set_request_timezone(self.region)
response = i_manager.dispatch(args,
"places",
instance_name=self.region)
else:
available_instances = get_all_available_instances(user)
autocomplete = global_autocomplete.get('bragi')
if not autocomplete:
raise TechnicalError(
'world wide autocompletion service not available')
response = autocomplete.get(args, instances=available_instances)
return response, 200
def get_street_network_db(self, instance, streetnetwork_backend_conf):
# type: (Instance, StreetNetworkBackend) -> Optional[AbstractStreetNetworkService]
# Make sure we update the streetnetwork_backends list from the database before returning them
self._update_config(instance)
sn = self._streetnetwork_backends.get(streetnetwork_backend_conf, None)
if sn is None:
raise TechnicalError(
'impossible to find a streetnetwork module for instance {} with configuration {}'
.format(instance, streetnetwork_backend_conf))
return sn
def get_street_network_routing_matrix(self, origins, destinations, mode, max_duration, request, **kwargs):
params = self.get_matrix_params(origins, destinations, mode, max_duration, request)
r = self._call_here(self.matrix_service_url, params=params)
if r.status_code != 200:
self.log.error('impossible to query HERE: {} with {} response: {}'
.format(r.url, r.status_code, r.text))
raise TechnicalError('invalid HERE call, impossible to query: {}'.format(r.url))
resp_json = r.json()
return self._get_matrix(resp_json, origins, destinations)
def get(self, id, region=None, lon=None, lat=None):
args = self.parsers["get"].parse_args()
args.update({
"uri": transform_id(id),
"_current_datetime": datetime.utcnow()
})
request_id = "places_{}".format(flask.request.id)
args["request_id"] = request_id
if any([region, lon, lat]):
self.region = i_manager.get_region(region, lon, lat)
timezone.set_request_timezone(self.region)
response = i_manager.dispatch(args,
"place_uri",
instance_name=self.region)
else:
user = authentication.get_user(token=authentication.get_token(),
abort_if_no_token=False)
available_instances = get_all_available_instances(
user, exclude_backend='kraken')
# If no instance available most probably due to database error
if (not user) and (not available_instances):
raise TechnicalError(
'world wide autocompletion service not available temporarily'
)
# If parameter '_autocomplete' is absent use 'bragi' as default value
if args["_autocomplete"] is None:
args["_autocomplete"] = app.config.get(
'DEFAULT_AUTOCOMPLETE_BACKEND', 'bragi')
autocomplete = global_autocomplete.get(args["_autocomplete"])
if not autocomplete:
raise TechnicalError(
'world wide autocompletion service not available')
response = autocomplete.get_by_uri(args["uri"],
request_id=request_id,
instances=available_instances)
return response, 200
def get_street_network(self, mode, request):
overriden_sn_id = request.get('_street_network')
if overriden_sn_id:
def predicate(s):
return s.sn_system_id == overriden_sn_id
else:
def predicate(s):
return mode in s.modes
sn = next((s for s in self.street_network_services if predicate(s)), None)
if sn is None:
raise TechnicalError('impossible to find a streetnetwork module for {} ({})'.format(
mode, overriden_sn_id))
return sn
def get_here_mode(mode):
map_mode = {
fm.walking.name: 'pedestrian',
fm.bike.name: 'bicycle',
fm.car.name: 'car',
fm.car_no_park.name: 'car',
fm.taxi.name: 'car',
fm.ridesharing.name: 'car',
}
try:
return map_mode[mode]
except KeyError:
raise TechnicalError('HERE does not handle the mode {}'.format(mode))
def get_street_network_routing_matrix(
self, instance, origins, destinations, mode, max_duration, request, **kwargs
):
# for now valhalla only manages 1-n request, so we reverse request if needed
if len(origins) > 1:
if len(destinations) > 1:
logging.getLogger(__name__).error('routing matrix error, no unique center point')
raise TechnicalError('routing matrix error, no unique center point')
data = self._make_request_arguments(mode, origins, destinations, request, api='sources_to_targets')
r = self._call_valhalla('{}/{}'.format(self.service_url, 'sources_to_targets'), requests.post, data)
self._check_response(r)
resp_json = r.json()
return self._get_matrix(resp_json, mode_park_cost=self.mode_park_cost.get(mode))
def _request():
with self.socket(self.instance.context) as socket:
socket.send(request.SerializeToString())
# timeout is in second, we need it on millisecond
if socket.poll(timeout=self.timeout * 1000) > 0:
pb = socket.recv()
resp = response_pb2.Response()
resp.ParseFromString(pb)
return resp
else:
socket.setsockopt(zmq.LINGER, 0)
socket.close()
self.logger.error('request on %s failed: %s', self.asgard_socket, six.text_type(request))
raise TechnicalError('asgard on {} failed'.format(self.asgard_socket))
def post_matrix_request(self, origins, destinations, request):
post_data = self.matrix_payload(origins, destinations, request)
params = {'apiKey': self.apiKey}
headers = {'Content-Type': 'application/json'}
post_resp = self._call_here(self.matrix_service_url,
params=params,
http_method=requests.post,
data=post_data,
headers=headers)
post_resp.raise_for_status()
if post_resp.status_code != 202:
raise TechnicalError('Here, post return status code != 202')
return post_resp.json()
def direct_path(self, mode, pt_object_origin, pt_object_destination,
datetime, clockwise):
url = self._format_url(mode, pt_object_origin, pt_object_destination)
r = self._call_valhalla(url)
if r == None:
raise TechnicalError('impossible to access valhalla service')
if r.status_code != 200:
logging.getLogger(__name__).error(
'Valhalla service unavailable, impossible to query : {}'.
format(r.url))
resp = response_pb2.Response()
resp.status_code = r.status_code
resp.error.message = 'Valhalla service unavailable, impossible to query : {}'.format(
r.url)
return resp
resp_json = r.json()
return self._get_response(resp_json, mode, pt_object_origin,
pt_object_destination, datetime)
def __init__(self,
context,
name,
zmq_socket,
street_network_configurations,
ridesharing_configurations,
realtime_proxies_configuration,
zmq_socket_type,
autocomplete_type):
self.geom = None
self._sockets = queue.Queue()
self.socket_path = zmq_socket
self._scenario = None
self._scenario_name = None
self.lock = Lock()
self.context = context
self.name = name
self.timezone = None # timezone will be fetched from the kraken
self.publication_date = -1
self.is_initialized = False #kraken hasn't been called yet we don't have geom nor timezone
self.breaker = pybreaker.CircuitBreaker(fail_max=app.config['CIRCUIT_BREAKER_MAX_INSTANCE_FAIL'],
reset_timeout=app.config['CIRCUIT_BREAKER_INSTANCE_TIMEOUT_S'])
self.georef = georef.Kraken(self)
self.planner = planner.Kraken(self)
street_network_configurations = _set_default_street_network_config(street_network_configurations)
self.street_network_services = street_network.StreetNetwork.get_street_network_services(self,
street_network_configurations)
self.ridesharing_services = []
if ridesharing_configurations is not None:
self.ridesharing_services = ridesharing_service.Ridesharing.\
get_ridesharing_services(self, ridesharing_configurations)
self.ptref = ptref.PtRef(self)
self.schedule = schedule.MixedSchedule(self)
self.realtime_proxy_manager = realtime_schedule.RealtimeProxyManager(realtime_proxies_configuration, self)
self.autocomplete = global_autocomplete.get(autocomplete_type)
if not self.autocomplete:
raise TechnicalError('impossible to find autocomplete system {} '
'cannot initialize instance {}'.format(autocomplete_type, name))
self.zmq_socket_type = zmq_socket_type
