class FakeInstance(Instance):
"""
The only purpose of this class is to override get_all_street_networks()
To bypass the app.config[str('DISABLE_DATABASE')] and the get_models() of the real implementation
"""
def __init__(
self,
disable_database,
ridesharing_configurations=None,
equipment_details_config=None,
instance_equipment_providers=None,
):
super(FakeInstance, self).__init__(
context=None,
name="instance",
zmq_socket=None,
street_network_configurations=[],
ridesharing_configurations=ridesharing_configurations,
instance_equipment_providers=[],
realtime_proxies_configuration=[],
pt_planners_configurations={},
zmq_socket_type=None,
autocomplete_type='kraken',
streetnetwork_backend_manager=StreetNetworkBackendManager(),
external_service_provider_configurations=[],
)
self.disable_database = disable_database
self.equipment_provider_manager = EquipmentProviderManager(
equipment_details_config)
self.equipment_provider_manager.init_providers(
instance_equipment_providers)
def get_models(self):
return None
def get_all_street_networks(self):
return (self.get_all_street_networks_json() if self.disable_database
else self.get_all_street_networks_db())
# returns a list
def get_all_street_networks_json(self):
return [krakenBss, krakenAll]
# returns a dict
def get_all_street_networks_db(self):
return {krakenBss: ["bss"], krakenAll: ["walking", "bike", "car"]}
class FakeInstance(Instance):
def __init__(
self,
disable_database,
ridesharing_configurations=None,
equipment_details_config=None,
instance_equipment_providers=None,
):
super(FakeInstance, self).__init__(
context=None,
name="instance",
zmq_socket=None,
street_network_configurations=[],
ridesharing_configurations=ridesharing_configurations,
instance_equipment_providers=[],
realtime_proxies_configuration=[],
zmq_socket_type=None,
autocomplete_type='kraken',
streetnetwork_backend_manager=None,
)
self.disable_database = disable_database
self.equipment_provider_manager = EquipmentProviderManager(
equipment_details_config)
self.equipment_provider_manager.init_providers(
instance_equipment_providers)
def get_models(self):
return None
def get_all_street_networks(self):
return (self.get_all_street_networks_json() if self.disable_database
else self.get_all_street_networks_db())
# returns a list
def get_all_street_networks_json(self):
return [krakenBss, krakenAll]
# returns a dict
def get_all_street_networks_db(self):
return {krakenBss: ["bss"], krakenAll: ["walking", "bike", "car"]}
class Instance(object):
name = None # type: Text
_sockets = None # type: Deque[Tuple[zmq.Socket, float]]
def __init__(
self,
context, # type: zmq.Context
name, # type: Text
zmq_socket, # type: Text
street_network_configurations,
ridesharing_configurations,
realtime_proxies_configuration,
zmq_socket_type,
autocomplete_type,
instance_equipment_providers, # type: List[Text]
streetnetwork_backend_manager,
):
self.geom = None
self.geojson = None
self._sockets = deque()
self.socket_path = zmq_socket
self._scenario = None
self._scenario_name = None
self.lock = Lock()
self.context = context
self.name = name
self.timezone = None # timezone will be fetched from the kraken
self.publication_date = -1
self.is_initialized = False # kraken hasn't been called yet we don't have geom nor timezone
self.breaker = pybreaker.CircuitBreaker(
fail_max=app.config.get(str('CIRCUIT_BREAKER_MAX_INSTANCE_FAIL'),
5),
reset_timeout=app.config.get(
str('CIRCUIT_BREAKER_INSTANCE_TIMEOUT_S'), 60),
)
self.georef = georef.Kraken(self)
self.planner = planner.Kraken(self)
self._streetnetwork_backend_manager = streetnetwork_backend_manager
if app.config[str('DISABLE_DATABASE')]:
self._streetnetwork_backend_manager.init_streetnetwork_backends_legacy(
self, street_network_configurations)
self.ridesharing_services = [
] # type: List[ridesharing_service.AbstractRidesharingService]
if ridesharing_configurations is not None:
self.ridesharing_services = ridesharing_service.Ridesharing.get_ridesharing_services(
self, ridesharing_configurations)
self.ptref = ptref.PtRef(self)
self.schedule = schedule.MixedSchedule(self)
self.realtime_proxy_manager = realtime_schedule.RealtimeProxyManager(
realtime_proxies_configuration, self)
self._autocomplete_type = autocomplete_type
if self._autocomplete_type is not None and self._autocomplete_type not in global_autocomplete:
raise RuntimeError('impossible to find autocomplete system {} '
'cannot initialize instance {}'.format(
autocomplete_type, name))
self.zmq_socket_type = zmq_socket_type
if app.config[str('DISABLE_DATABASE')]:
self.equipment_provider_manager = EquipmentProviderManager(
app.config[str('EQUIPMENT_DETAILS_PROVIDERS')])
else:
self.equipment_provider_manager = EquipmentProviderManager(
app.config[str('EQUIPMENT_DETAILS_PROVIDERS')],
self.get_providers_from_db)
# Init equipment providers from config
self.equipment_provider_manager.init_providers(
instance_equipment_providers)
def get_providers_from_db(self):
"""
:return: a callable query of equipment providers associated to the current instance in db
"""
return self._get_models().equipment_details_providers
@property
def autocomplete(self):
if self._autocomplete_type:
# retrocompat: we need to continue to read configuration from file
# while we migrate to database configuration
return global_autocomplete.get(self._autocomplete_type)
backend = global_autocomplete.get(self.autocomplete_backend)
if backend is None:
raise RuntimeError(
'impossible to find autocomplete {} for instance {}'.format(
self.autocomplete_backend, self.name))
return backend
def stop_point_fallbacks(self):
return [
a for a in global_autocomplete.values()
if a.is_handling_stop_points()
]
def get_models(self):
if self.name not in g.instances_model:
g.instances_model[self.name] = self._get_models()
return g.instances_model[self.name]
def __repr__(self):
return 'instance.{}'.format(self.name)
@memory_cache.memoize(app.config[str('MEMORY_CACHE_CONFIGURATION')].get(
str('TIMEOUT_PARAMS'), 30))
@cache.memoize(app.config[str('CACHE_CONFIGURATION')].get(
str('TIMEOUT_PARAMS'), 300))
def _get_models(self):
if app.config['DISABLE_DATABASE']:
return None
return models.Instance.get_by_name(self.name)
def scenario(self, override_scenario=None):
if hasattr(g, 'scenario') and g.scenario:
"""
once a scenario has been chosen for a request, we cannot change it
"""
return g.scenario
def replace_experimental_scenario(s):
return 'distributed' if s == 'experimental' else s
if override_scenario:
logger = logging.getLogger(__name__)
logger.debug('overriding the scenario for %s with %s', self.name,
override_scenario)
try:
# for the sake of backwards compatibility... some users may still be using experimental...
override_scenario = replace_experimental_scenario(
override_scenario)
module = import_module(
'jormungandr.scenarios.{}'.format(override_scenario))
except ImportError:
logger.exception('scenario not found')
abort(404,
message='invalid scenario: {}'.format(override_scenario))
scenario = module.Scenario()
g.scenario = scenario
return scenario
instance_db = self.get_models()
scenario_name = instance_db.scenario if instance_db else 'new_default'
# for the sake of backwards compatibility... some users may still be using experimental...
scenario_name = replace_experimental_scenario(scenario_name)
if not self._scenario or scenario_name != self._scenario_name:
logger = logging.getLogger(__name__)
logger.info('loading of scenario %s for instance %s',
scenario_name, self.name)
self._scenario_name = scenario_name
module = import_module(
'jormungandr.scenarios.{}'.format(scenario_name))
self._scenario = module.Scenario()
# we save the used scenario for future use
g.scenario = self._scenario
return self._scenario
@property
def journey_order(self):
# type: () -> Text
instance_db = self.get_models()
return get_value_or_default('journey_order', instance_db, self.name)
@property
def autocomplete_backend(self):
# type: () -> Text
instance_db = self.get_models()
return get_value_or_default('autocomplete_backend', instance_db,
self.name)
@property
def max_walking_duration_to_pt(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('max_walking_duration_to_pt', instance_db,
self.name)
@property
def max_bss_duration_to_pt(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('max_bss_duration_to_pt', instance_db,
self.name)
@property
def max_bike_duration_to_pt(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('max_bike_duration_to_pt', instance_db,
self.name)
@property
def max_car_duration_to_pt(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('max_car_duration_to_pt', instance_db,
self.name)
@property
def max_car_no_park_duration_to_pt(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('max_car_no_park_duration_to_pt',
instance_db, self.name)
@property
def walking_speed(self):
# type: () -> float
instance_db = self.get_models()
return get_value_or_default('walking_speed', instance_db, self.name)
@property
def bss_speed(self):
# type: () -> float
instance_db = self.get_models()
return get_value_or_default('bss_speed', instance_db, self.name)
@property
def bike_speed(self):
# type: () -> float
instance_db = self.get_models()
return get_value_or_default('bike_speed', instance_db, self.name)
@property
def car_speed(self):
# type: () -> float
instance_db = self.get_models()
return get_value_or_default('car_speed', instance_db, self.name)
@property
def car_no_park_speed(self):
# type: () -> float
instance_db = self.get_models()
return get_value_or_default('car_no_park_speed', instance_db,
self.name)
@property
def max_nb_transfers(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('max_nb_transfers', instance_db, self.name)
@property
def min_bike(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('min_bike', instance_db, self.name)
@property
def min_bss(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('min_bss', instance_db, self.name)
@property
def min_car(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('min_car', instance_db, self.name)
@property
def min_taxi(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('min_taxi', instance_db, self.name)
@property
def successive_physical_mode_to_limit_id(self):
# type: () -> Text
instance_db = self.get_models()
return get_value_or_default('successive_physical_mode_to_limit_id',
instance_db, self.name)
@property
def priority(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('priority', instance_db, self.name)
@property
def bss_provider(self):
# type: () -> bool
instance_db = self.get_models()
return get_value_or_default('bss_provider', instance_db, self.name)
@property
def car_park_provider(self):
# type: () -> bool
instance_db = self.get_models()
return get_value_or_default('car_park_provider', instance_db,
self.name)
@property
def max_additional_connections(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('max_additional_connections', instance_db,
self.name)
@property
def is_free(self):
# type: () -> bool
instance_db = self.get_models()
if not instance_db:
return False
else:
return instance_db.is_free
@property
def is_open_data(self):
# type: () -> bool
instance_db = self.get_models()
if not instance_db:
return False
else:
return instance_db.is_open_data
@property
def max_duration(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('max_duration', instance_db, self.name)
@property
def walking_transfer_penalty(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('walking_transfer_penalty', instance_db,
self.name)
@property
def night_bus_filter_max_factor(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('night_bus_filter_max_factor', instance_db,
self.name)
@property
def night_bus_filter_base_factor(self):
# type: () -> float
instance_db = self.get_models()
return get_value_or_default('night_bus_filter_base_factor',
instance_db, self.name)
@property
def realtime_pool_size(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('realtime_pool_size', instance_db,
self.name)
@property
def min_nb_journeys(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('min_nb_journeys', instance_db, self.name)
@property
def max_nb_journeys(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('max_nb_journeys', instance_db, self.name)
@property
def min_journeys_calls(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('min_journeys_calls', instance_db,
self.name)
@property
def max_successive_physical_mode(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('max_successive_physical_mode',
instance_db, self.name)
@property
def final_line_filter(self):
instance_db = self.get_models()
return get_value_or_default('final_line_filter', instance_db,
self.name)
@property
def max_extra_second_pass(self):
# type: () -> int
instance_db = self.get_models()
return get_value_or_default('max_extra_second_pass', instance_db,
self.name)
@property
def max_nb_crowfly_by_mode(self):
# type: () -> Dict[Text, int]
instance_db = self.get_models()
# the value by default is a dict...
d = copy.deepcopy(
get_value_or_default('max_nb_crowfly_by_mode', instance_db,
self.name))
# In case we add a new max_nb_crowfly for an other mode than
# the ones already present in the database.
for mode, duration in default_values.max_nb_crowfly_by_mode.items():
if mode not in d:
d[mode] = duration
return d
@property
def poi_dataset(self):
# type: () -> Text
instance_db = self.get_models()
return instance_db.poi_dataset if instance_db else None
# TODO: refactorise all properties
taxi_speed = _make_property_getter('taxi_speed')
additional_time_after_first_section_taxi = _make_property_getter(
'additional_time_after_first_section_taxi')
additional_time_before_last_section_taxi = _make_property_getter(
'additional_time_before_last_section_taxi')
max_walking_direct_path_duration = _make_property_getter(
'max_walking_direct_path_duration')
max_bike_direct_path_duration = _make_property_getter(
'max_bike_direct_path_duration')
max_bss_direct_path_duration = _make_property_getter(
'max_bss_direct_path_duration')
max_car_direct_path_duration = _make_property_getter(
'max_car_direct_path_duration')
max_taxi_direct_path_duration = _make_property_getter(
'max_taxi_direct_path_duration')
max_ridesharing_direct_path_duration = _make_property_getter(
'max_ridesharing_direct_path_duration')
street_network_car = _make_property_getter('street_network_car')
street_network_walking = _make_property_getter('street_network_walking')
street_network_bike = _make_property_getter('street_network_bike')
street_network_bss = _make_property_getter('street_network_bss')
street_network_ridesharing = _make_property_getter(
'street_network_ridesharing')
street_network_taxi = _make_property_getter('street_network_taxi')
def reap_socket(self, ttl):
# type: (int) -> None
if self.zmq_socket_type != 'transient':
return
logger = logging.getLogger(__name__)
now = time.time()
while True:
try:
socket, t = self._sockets.popleft()
if now - t > ttl:
logger.debug("closing one socket for %s", self.name)
socket.setsockopt(zmq.LINGER, 0)
socket.close()
else:
self._sockets.appendleft((socket, t))
break # remaining socket are still in "keep alive" state
except IndexError:
break
@contextmanager
def socket(self, context):
socket = None
try:
socket, _ = self._sockets.pop()
except IndexError: # there is no socket available: lets create one
socket = context.socket(zmq.REQ)
socket.connect(self.socket_path)
try:
yield socket
finally:
if not socket.closed:
self._sockets.append((socket, time.time()))
def send_and_receive(self, *args, **kwargs):
"""
encapsulate all call to kraken in a circuit breaker, this way we don't loose time calling dead instance
"""
try:
return self.breaker.call(self._send_and_receive, *args, **kwargs)
except pybreaker.CircuitBreakerError as e:
raise DeadSocketException(self.name, self.socket_path)
def _send_and_receive(self,
request,
timeout=app.config.get('INSTANCE_TIMEOUT', 10000),
quiet=False,
**kwargs):
logger = logging.getLogger(__name__)
deadline = datetime.utcnow() + timedelta(milliseconds=timeout)
request.deadline = deadline.strftime('%Y%m%dT%H%M%S,%f')
with self.socket(self.context) as socket:
try:
request.request_id = flask.request.id
except RuntimeError:
# we aren't in a flask context, so there is no request
if 'flask_request_id' in kwargs:
request.request_id = kwargs['flask_request_id']
socket.send(request.SerializeToString())
if socket.poll(timeout=timeout) > 0:
pb = socket.recv()
resp = response_pb2.Response()
resp.ParseFromString(pb)
self.update_property(
resp
) # we update the timezone and geom of the instances at each request
return resp
else:
socket.setsockopt(zmq.LINGER, 0)
socket.close()
if not quiet:
logger.error('request on %s failed: %s', self.socket_path,
six.text_type(request))
raise DeadSocketException(self.name, self.socket_path)
def get_id(self, id_):
"""
Get the pt_object that have the given id
"""
req = request_pb2.Request()
req.requested_api = type_pb2.place_uri
req.place_uri.uri = id_
return self.send_and_receive(req,
timeout=app.config.get(
'INSTANCE_FAST_TIMEOUT', 1000))
def has_id(self, id_):
"""
Does this instance has this id
"""
try:
return len(self.get_id(id_).places) > 0
except DeadSocketException:
return False
def has_coord(self, lon, lat):
return self.has_point(geometry.Point(lon, lat))
def has_point(self, p):
try:
return self.geom and self.geom.contains(p)
except DeadSocketException:
return False
except PredicateError:
logging.getLogger(__name__).exception("has_coord failed")
return False
def get_external_codes(self, type_, id_):
"""
Get all pt_object with the given id
"""
req = request_pb2.Request()
req.requested_api = type_pb2.place_code
if type_ not in type_to_pttype:
raise ValueError(
"Can't call pt_code API with type: {}".format(type_))
req.place_code.type = type_to_pttype[type_]
req.place_code.type_code = "external_code"
req.place_code.code = id_
# we set the timeout to 1s
return self.send_and_receive(req,
timeout=app.config.get(
'INSTANCE_FAST_TIMEOUT', 1000))
def has_external_code(self, type_, id_):
"""
Does this instance has the given id
Returns None if it doesnt, the kraken uri otherwise
"""
try:
res = self.get_external_codes(type_, id_)
except DeadSocketException:
return False
if len(res.places) > 0:
return res.places[0].uri
return None
def update_property(self, response):
"""
update the property of an instance from a response if the metadatas field if present
"""
# after a successful call we consider the instance initialised even if no data were loaded
self.is_initialized = True
if response.HasField(
str("metadatas"
)) and response.publication_date != self.publication_date:
logging.getLogger(__name__).debug('updating metadata for %s',
self.name)
with self.lock as lock:
self.publication_date = response.publication_date
if response.metadatas.shape and response.metadatas.shape != "":
try:
self.geom = wkt.loads(response.metadatas.shape)
except ReadingError:
self.geom = None
else:
self.geom = None
self.timezone = response.metadatas.timezone
self._update_geojson()
set_request_instance_timezone(self)
def _update_geojson(self):
"""construct the geojson object from the shape"""
if not self.geom or not self.geom.is_valid:
self.geojson = None
return
# simplify the geom to prevent slow query on bragi
geom = self.geom.simplify(tolerance=0.1)
self.geojson = geometry.mapping(geom)
def init(self):
"""
Get and store variables of the instance.
Returns True if we need to clear the cache, False otherwise.
"""
pub_date = self.publication_date
req = request_pb2.Request()
req.requested_api = type_pb2.METADATAS
try:
# we use _send_and_receive to avoid the circuit breaker, we don't want fast fail on init :)
resp = self._send_and_receive(req, timeout=1000, quiet=True)
# the instance is automatically updated on a call
if self.publication_date != pub_date:
return True
except DeadSocketException:
# we don't do anything on error, a new session will be established to an available kraken on
# the next request. We don't want to purge all our cache for a small error.
logging.getLogger(__name__).debug('timeout on init for %s',
self.name)
return False
def get_street_network(self, mode, request):
if app.config[str('DISABLE_DATABASE')]:
return self._streetnetwork_backend_manager.get_street_network_legacy(
self, mode, request)
else:
# We get the name of the column in the database corresponding to the mode used in the request
# And we get the value of this column for this instance
column_in_db = "street_network_{}".format(mode)
streetnetwork_backend_conf = getattr(self, column_in_db)
return self._streetnetwork_backend_manager.get_street_network_db(
self, streetnetwork_backend_conf)
def get_all_street_networks(self):
if app.config[str('DISABLE_DATABASE')]:
return self._streetnetwork_backend_manager.get_all_street_networks_legacy(
self)
else:
return self._streetnetwork_backend_manager.get_all_street_networks_db(
self)
def get_autocomplete(self, requested_autocomplete):
if not requested_autocomplete:
return self.autocomplete
autocomplete = global_autocomplete.get(requested_autocomplete)
if not autocomplete:
raise TechnicalError(
'autocomplete {} not available'.format(requested_autocomplete))
return autocomplete
def get_ridesharing_journeys_with_feed_publishers(self,
from_coord,
to_coord,
period_extremity,
limit=None):
res = []
fps = set()
for service in self.ridesharing_services:
rsjs, fp = service.request_journeys_with_feed_publisher(
from_coord, to_coord, period_extremity, limit)
res.extend(rsjs)
fps.add(fp)
return res, fps