class FakeInstance(Instance): """ The only purpose of this class is to override get_all_street_networks() To bypass the app.config[str('DISABLE_DATABASE')] and the get_models() of the real implementation """ def __init__( self, disable_database, ridesharing_configurations=None, equipment_details_config=None, instance_equipment_providers=None, ): super(FakeInstance, self).__init__( context=None, name="instance", zmq_socket=None, street_network_configurations=[], ridesharing_configurations=ridesharing_configurations, instance_equipment_providers=[], realtime_proxies_configuration=[], pt_planners_configurations={}, zmq_socket_type=None, autocomplete_type='kraken', streetnetwork_backend_manager=StreetNetworkBackendManager(), external_service_provider_configurations=[], ) self.disable_database = disable_database self.equipment_provider_manager = EquipmentProviderManager( equipment_details_config) self.equipment_provider_manager.init_providers( instance_equipment_providers) def get_models(self): return None def get_all_street_networks(self): return (self.get_all_street_networks_json() if self.disable_database else self.get_all_street_networks_db()) # returns a list def get_all_street_networks_json(self): return [krakenBss, krakenAll] # returns a dict def get_all_street_networks_db(self): return {krakenBss: ["bss"], krakenAll: ["walking", "bike", "car"]}
class FakeInstance(Instance): def __init__( self, disable_database, ridesharing_configurations=None, equipment_details_config=None, instance_equipment_providers=None, ): super(FakeInstance, self).__init__( context=None, name="instance", zmq_socket=None, street_network_configurations=[], ridesharing_configurations=ridesharing_configurations, instance_equipment_providers=[], realtime_proxies_configuration=[], zmq_socket_type=None, autocomplete_type='kraken', streetnetwork_backend_manager=None, ) self.disable_database = disable_database self.equipment_provider_manager = EquipmentProviderManager( equipment_details_config) self.equipment_provider_manager.init_providers( instance_equipment_providers) def get_models(self): return None def get_all_street_networks(self): return (self.get_all_street_networks_json() if self.disable_database else self.get_all_street_networks_db()) # returns a list def get_all_street_networks_json(self): return [krakenBss, krakenAll] # returns a dict def get_all_street_networks_db(self): return {krakenBss: ["bss"], krakenAll: ["walking", "bike", "car"]}
class Instance(object): name = None # type: Text _sockets = None # type: Deque[Tuple[zmq.Socket, float]] def __init__( self, context, # type: zmq.Context name, # type: Text zmq_socket, # type: Text street_network_configurations, ridesharing_configurations, realtime_proxies_configuration, zmq_socket_type, autocomplete_type, instance_equipment_providers, # type: List[Text] streetnetwork_backend_manager, ): self.geom = None self.geojson = None self._sockets = deque() self.socket_path = zmq_socket self._scenario = None self._scenario_name = None self.lock = Lock() self.context = context self.name = name self.timezone = None # timezone will be fetched from the kraken self.publication_date = -1 self.is_initialized = False # kraken hasn't been called yet we don't have geom nor timezone self.breaker = pybreaker.CircuitBreaker( fail_max=app.config.get(str('CIRCUIT_BREAKER_MAX_INSTANCE_FAIL'), 5), reset_timeout=app.config.get( str('CIRCUIT_BREAKER_INSTANCE_TIMEOUT_S'), 60), ) self.georef = georef.Kraken(self) self.planner = planner.Kraken(self) self._streetnetwork_backend_manager = streetnetwork_backend_manager if app.config[str('DISABLE_DATABASE')]: self._streetnetwork_backend_manager.init_streetnetwork_backends_legacy( self, street_network_configurations) self.ridesharing_services = [ ] # type: List[ridesharing_service.AbstractRidesharingService] if ridesharing_configurations is not None: self.ridesharing_services = ridesharing_service.Ridesharing.get_ridesharing_services( self, ridesharing_configurations) self.ptref = ptref.PtRef(self) self.schedule = schedule.MixedSchedule(self) self.realtime_proxy_manager = realtime_schedule.RealtimeProxyManager( realtime_proxies_configuration, self) self._autocomplete_type = autocomplete_type if self._autocomplete_type is not None and self._autocomplete_type not in global_autocomplete: raise RuntimeError('impossible to find autocomplete system {} ' 'cannot initialize instance {}'.format( autocomplete_type, name)) self.zmq_socket_type = zmq_socket_type if app.config[str('DISABLE_DATABASE')]: self.equipment_provider_manager = EquipmentProviderManager( app.config[str('EQUIPMENT_DETAILS_PROVIDERS')]) else: self.equipment_provider_manager = EquipmentProviderManager( app.config[str('EQUIPMENT_DETAILS_PROVIDERS')], self.get_providers_from_db) # Init equipment providers from config self.equipment_provider_manager.init_providers( instance_equipment_providers) def get_providers_from_db(self): """ :return: a callable query of equipment providers associated to the current instance in db """ return self._get_models().equipment_details_providers @property def autocomplete(self): if self._autocomplete_type: # retrocompat: we need to continue to read configuration from file # while we migrate to database configuration return global_autocomplete.get(self._autocomplete_type) backend = global_autocomplete.get(self.autocomplete_backend) if backend is None: raise RuntimeError( 'impossible to find autocomplete {} for instance {}'.format( self.autocomplete_backend, self.name)) return backend def stop_point_fallbacks(self): return [ a for a in global_autocomplete.values() if a.is_handling_stop_points() ] def get_models(self): if self.name not in g.instances_model: g.instances_model[self.name] = self._get_models() return g.instances_model[self.name] def __repr__(self): return 'instance.{}'.format(self.name) @memory_cache.memoize(app.config[str('MEMORY_CACHE_CONFIGURATION')].get( str('TIMEOUT_PARAMS'), 30)) @cache.memoize(app.config[str('CACHE_CONFIGURATION')].get( str('TIMEOUT_PARAMS'), 300)) def _get_models(self): if app.config['DISABLE_DATABASE']: return None return models.Instance.get_by_name(self.name) def scenario(self, override_scenario=None): if hasattr(g, 'scenario') and g.scenario: """ once a scenario has been chosen for a request, we cannot change it """ return g.scenario def replace_experimental_scenario(s): return 'distributed' if s == 'experimental' else s if override_scenario: logger = logging.getLogger(__name__) logger.debug('overriding the scenario for %s with %s', self.name, override_scenario) try: # for the sake of backwards compatibility... some users may still be using experimental... override_scenario = replace_experimental_scenario( override_scenario) module = import_module( 'jormungandr.scenarios.{}'.format(override_scenario)) except ImportError: logger.exception('scenario not found') abort(404, message='invalid scenario: {}'.format(override_scenario)) scenario = module.Scenario() g.scenario = scenario return scenario instance_db = self.get_models() scenario_name = instance_db.scenario if instance_db else 'new_default' # for the sake of backwards compatibility... some users may still be using experimental... scenario_name = replace_experimental_scenario(scenario_name) if not self._scenario or scenario_name != self._scenario_name: logger = logging.getLogger(__name__) logger.info('loading of scenario %s for instance %s', scenario_name, self.name) self._scenario_name = scenario_name module = import_module( 'jormungandr.scenarios.{}'.format(scenario_name)) self._scenario = module.Scenario() # we save the used scenario for future use g.scenario = self._scenario return self._scenario @property def journey_order(self): # type: () -> Text instance_db = self.get_models() return get_value_or_default('journey_order', instance_db, self.name) @property def autocomplete_backend(self): # type: () -> Text instance_db = self.get_models() return get_value_or_default('autocomplete_backend', instance_db, self.name) @property def max_walking_duration_to_pt(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('max_walking_duration_to_pt', instance_db, self.name) @property def max_bss_duration_to_pt(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('max_bss_duration_to_pt', instance_db, self.name) @property def max_bike_duration_to_pt(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('max_bike_duration_to_pt', instance_db, self.name) @property def max_car_duration_to_pt(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('max_car_duration_to_pt', instance_db, self.name) @property def max_car_no_park_duration_to_pt(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('max_car_no_park_duration_to_pt', instance_db, self.name) @property def walking_speed(self): # type: () -> float instance_db = self.get_models() return get_value_or_default('walking_speed', instance_db, self.name) @property def bss_speed(self): # type: () -> float instance_db = self.get_models() return get_value_or_default('bss_speed', instance_db, self.name) @property def bike_speed(self): # type: () -> float instance_db = self.get_models() return get_value_or_default('bike_speed', instance_db, self.name) @property def car_speed(self): # type: () -> float instance_db = self.get_models() return get_value_or_default('car_speed', instance_db, self.name) @property def car_no_park_speed(self): # type: () -> float instance_db = self.get_models() return get_value_or_default('car_no_park_speed', instance_db, self.name) @property def max_nb_transfers(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('max_nb_transfers', instance_db, self.name) @property def min_bike(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('min_bike', instance_db, self.name) @property def min_bss(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('min_bss', instance_db, self.name) @property def min_car(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('min_car', instance_db, self.name) @property def min_taxi(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('min_taxi', instance_db, self.name) @property def successive_physical_mode_to_limit_id(self): # type: () -> Text instance_db = self.get_models() return get_value_or_default('successive_physical_mode_to_limit_id', instance_db, self.name) @property def priority(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('priority', instance_db, self.name) @property def bss_provider(self): # type: () -> bool instance_db = self.get_models() return get_value_or_default('bss_provider', instance_db, self.name) @property def car_park_provider(self): # type: () -> bool instance_db = self.get_models() return get_value_or_default('car_park_provider', instance_db, self.name) @property def max_additional_connections(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('max_additional_connections', instance_db, self.name) @property def is_free(self): # type: () -> bool instance_db = self.get_models() if not instance_db: return False else: return instance_db.is_free @property def is_open_data(self): # type: () -> bool instance_db = self.get_models() if not instance_db: return False else: return instance_db.is_open_data @property def max_duration(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('max_duration', instance_db, self.name) @property def walking_transfer_penalty(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('walking_transfer_penalty', instance_db, self.name) @property def night_bus_filter_max_factor(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('night_bus_filter_max_factor', instance_db, self.name) @property def night_bus_filter_base_factor(self): # type: () -> float instance_db = self.get_models() return get_value_or_default('night_bus_filter_base_factor', instance_db, self.name) @property def realtime_pool_size(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('realtime_pool_size', instance_db, self.name) @property def min_nb_journeys(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('min_nb_journeys', instance_db, self.name) @property def max_nb_journeys(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('max_nb_journeys', instance_db, self.name) @property def min_journeys_calls(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('min_journeys_calls', instance_db, self.name) @property def max_successive_physical_mode(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('max_successive_physical_mode', instance_db, self.name) @property def final_line_filter(self): instance_db = self.get_models() return get_value_or_default('final_line_filter', instance_db, self.name) @property def max_extra_second_pass(self): # type: () -> int instance_db = self.get_models() return get_value_or_default('max_extra_second_pass', instance_db, self.name) @property def max_nb_crowfly_by_mode(self): # type: () -> Dict[Text, int] instance_db = self.get_models() # the value by default is a dict... d = copy.deepcopy( get_value_or_default('max_nb_crowfly_by_mode', instance_db, self.name)) # In case we add a new max_nb_crowfly for an other mode than # the ones already present in the database. for mode, duration in default_values.max_nb_crowfly_by_mode.items(): if mode not in d: d[mode] = duration return d @property def poi_dataset(self): # type: () -> Text instance_db = self.get_models() return instance_db.poi_dataset if instance_db else None # TODO: refactorise all properties taxi_speed = _make_property_getter('taxi_speed') additional_time_after_first_section_taxi = _make_property_getter( 'additional_time_after_first_section_taxi') additional_time_before_last_section_taxi = _make_property_getter( 'additional_time_before_last_section_taxi') max_walking_direct_path_duration = _make_property_getter( 'max_walking_direct_path_duration') max_bike_direct_path_duration = _make_property_getter( 'max_bike_direct_path_duration') max_bss_direct_path_duration = _make_property_getter( 'max_bss_direct_path_duration') max_car_direct_path_duration = _make_property_getter( 'max_car_direct_path_duration') max_taxi_direct_path_duration = _make_property_getter( 'max_taxi_direct_path_duration') max_ridesharing_direct_path_duration = _make_property_getter( 'max_ridesharing_direct_path_duration') street_network_car = _make_property_getter('street_network_car') street_network_walking = _make_property_getter('street_network_walking') street_network_bike = _make_property_getter('street_network_bike') street_network_bss = _make_property_getter('street_network_bss') street_network_ridesharing = _make_property_getter( 'street_network_ridesharing') street_network_taxi = _make_property_getter('street_network_taxi') def reap_socket(self, ttl): # type: (int) -> None if self.zmq_socket_type != 'transient': return logger = logging.getLogger(__name__) now = time.time() while True: try: socket, t = self._sockets.popleft() if now - t > ttl: logger.debug("closing one socket for %s", self.name) socket.setsockopt(zmq.LINGER, 0) socket.close() else: self._sockets.appendleft((socket, t)) break # remaining socket are still in "keep alive" state except IndexError: break @contextmanager def socket(self, context): socket = None try: socket, _ = self._sockets.pop() except IndexError: # there is no socket available: lets create one socket = context.socket(zmq.REQ) socket.connect(self.socket_path) try: yield socket finally: if not socket.closed: self._sockets.append((socket, time.time())) def send_and_receive(self, *args, **kwargs): """ encapsulate all call to kraken in a circuit breaker, this way we don't loose time calling dead instance """ try: return self.breaker.call(self._send_and_receive, *args, **kwargs) except pybreaker.CircuitBreakerError as e: raise DeadSocketException(self.name, self.socket_path) def _send_and_receive(self, request, timeout=app.config.get('INSTANCE_TIMEOUT', 10000), quiet=False, **kwargs): logger = logging.getLogger(__name__) deadline = datetime.utcnow() + timedelta(milliseconds=timeout) request.deadline = deadline.strftime('%Y%m%dT%H%M%S,%f') with self.socket(self.context) as socket: try: request.request_id = flask.request.id except RuntimeError: # we aren't in a flask context, so there is no request if 'flask_request_id' in kwargs: request.request_id = kwargs['flask_request_id'] socket.send(request.SerializeToString()) if socket.poll(timeout=timeout) > 0: pb = socket.recv() resp = response_pb2.Response() resp.ParseFromString(pb) self.update_property( resp ) # we update the timezone and geom of the instances at each request return resp else: socket.setsockopt(zmq.LINGER, 0) socket.close() if not quiet: logger.error('request on %s failed: %s', self.socket_path, six.text_type(request)) raise DeadSocketException(self.name, self.socket_path) def get_id(self, id_): """ Get the pt_object that have the given id """ req = request_pb2.Request() req.requested_api = type_pb2.place_uri req.place_uri.uri = id_ return self.send_and_receive(req, timeout=app.config.get( 'INSTANCE_FAST_TIMEOUT', 1000)) def has_id(self, id_): """ Does this instance has this id """ try: return len(self.get_id(id_).places) > 0 except DeadSocketException: return False def has_coord(self, lon, lat): return self.has_point(geometry.Point(lon, lat)) def has_point(self, p): try: return self.geom and self.geom.contains(p) except DeadSocketException: return False except PredicateError: logging.getLogger(__name__).exception("has_coord failed") return False def get_external_codes(self, type_, id_): """ Get all pt_object with the given id """ req = request_pb2.Request() req.requested_api = type_pb2.place_code if type_ not in type_to_pttype: raise ValueError( "Can't call pt_code API with type: {}".format(type_)) req.place_code.type = type_to_pttype[type_] req.place_code.type_code = "external_code" req.place_code.code = id_ # we set the timeout to 1s return self.send_and_receive(req, timeout=app.config.get( 'INSTANCE_FAST_TIMEOUT', 1000)) def has_external_code(self, type_, id_): """ Does this instance has the given id Returns None if it doesnt, the kraken uri otherwise """ try: res = self.get_external_codes(type_, id_) except DeadSocketException: return False if len(res.places) > 0: return res.places[0].uri return None def update_property(self, response): """ update the property of an instance from a response if the metadatas field if present """ # after a successful call we consider the instance initialised even if no data were loaded self.is_initialized = True if response.HasField( str("metadatas" )) and response.publication_date != self.publication_date: logging.getLogger(__name__).debug('updating metadata for %s', self.name) with self.lock as lock: self.publication_date = response.publication_date if response.metadatas.shape and response.metadatas.shape != "": try: self.geom = wkt.loads(response.metadatas.shape) except ReadingError: self.geom = None else: self.geom = None self.timezone = response.metadatas.timezone self._update_geojson() set_request_instance_timezone(self) def _update_geojson(self): """construct the geojson object from the shape""" if not self.geom or not self.geom.is_valid: self.geojson = None return # simplify the geom to prevent slow query on bragi geom = self.geom.simplify(tolerance=0.1) self.geojson = geometry.mapping(geom) def init(self): """ Get and store variables of the instance. Returns True if we need to clear the cache, False otherwise. """ pub_date = self.publication_date req = request_pb2.Request() req.requested_api = type_pb2.METADATAS try: # we use _send_and_receive to avoid the circuit breaker, we don't want fast fail on init :) resp = self._send_and_receive(req, timeout=1000, quiet=True) # the instance is automatically updated on a call if self.publication_date != pub_date: return True except DeadSocketException: # we don't do anything on error, a new session will be established to an available kraken on # the next request. We don't want to purge all our cache for a small error. logging.getLogger(__name__).debug('timeout on init for %s', self.name) return False def get_street_network(self, mode, request): if app.config[str('DISABLE_DATABASE')]: return self._streetnetwork_backend_manager.get_street_network_legacy( self, mode, request) else: # We get the name of the column in the database corresponding to the mode used in the request # And we get the value of this column for this instance column_in_db = "street_network_{}".format(mode) streetnetwork_backend_conf = getattr(self, column_in_db) return self._streetnetwork_backend_manager.get_street_network_db( self, streetnetwork_backend_conf) def get_all_street_networks(self): if app.config[str('DISABLE_DATABASE')]: return self._streetnetwork_backend_manager.get_all_street_networks_legacy( self) else: return self._streetnetwork_backend_manager.get_all_street_networks_db( self) def get_autocomplete(self, requested_autocomplete): if not requested_autocomplete: return self.autocomplete autocomplete = global_autocomplete.get(requested_autocomplete) if not autocomplete: raise TechnicalError( 'autocomplete {} not available'.format(requested_autocomplete)) return autocomplete def get_ridesharing_journeys_with_feed_publishers(self, from_coord, to_coord, period_extremity, limit=None): res = [] fps = set() for service in self.ridesharing_services: rsjs, fp = service.request_journeys_with_feed_publisher( from_coord, to_coord, period_extremity, limit) res.extend(rsjs) fps.add(fp) return res, fps