def __init__(self, id, service_url, service_args, timezone, object_id_tag=None, destination_id_tag=None, instance=None, timeout=10, **kwargs): self.service_url = service_url self.service_args = service_args self.timeout = timeout #timeout in seconds self.rt_system_id = id self.object_id_tag = object_id_tag if object_id_tag else id self.destination_id_tag = destination_id_tag self.instance = instance self.breaker = pybreaker.CircuitBreaker(fail_max=app.config['CIRCUIT_BREAKER_MAX_TIMEO_FAIL'], reset_timeout=app.config['CIRCUIT_BREAKER_TIMEO_TIMEOUT_S']) # Note: if the timezone is not know, pytz raise an error self.timezone = pytz.timezone(timezone) if kwargs.get('redis_host') and kwargs.get('rate_limit_count'): self.rate_limiter = RateLimiter(conditions=[{'requests': kwargs.get('rate_limit_count'), 'seconds': kwargs.get('rate_limit_duration', 1)}], redis_host=kwargs.get('redis_host'), redis_port=kwargs.get('redis_port', 6379), redis_db=kwargs.get('redis_db', 0), redis_password=kwargs.get('redis_password'), redis_namespace=kwargs.get('redis_namespace', 'jormungandr.rate_limiter')) else: self.rate_limiter = FakeRateLimiter()
def __init__(self, id, service_url, service_args, timezone, object_id_tag=None, destination_id_tag=None, instance=None, timeout=10, **kwargs): self.service_url = service_url self.service_args = service_args self.timeout = timeout #timeout in seconds self.rt_system_id = id self.object_id_tag = object_id_tag if object_id_tag else id self.destination_id_tag = destination_id_tag self.instance = instance fail_max = kwargs.get('circuit_breaker_max_fail', app.config['CIRCUIT_BREAKER_MAX_TIMEO_FAIL']) reset_timeout = kwargs.get('circuit_breaker_reset_timeout', app.config['CIRCUIT_BREAKER_TIMEO_TIMEOUT_S']) self.breaker = pybreaker.CircuitBreaker(fail_max=fail_max, reset_timeout=reset_timeout) # A step is applied on from_datetime to discretize calls and allow caching them self.from_datetime_step = kwargs.get('from_datetime_step', app.config['CACHE_CONFIGURATION'].get('TIMEOUT_TIMEO', 60)) # Note: if the timezone is not know, pytz raise an error self.timezone = pytz.timezone(timezone) if kwargs.get('redis_host') and kwargs.get('rate_limit_count'): self.rate_limiter = RateLimiter(conditions=[{'requests': kwargs.get('rate_limit_count'), 'seconds': kwargs.get('rate_limit_duration', 1)}], redis_host=kwargs.get('redis_host'), redis_port=kwargs.get('redis_port', 6379), redis_db=kwargs.get('redis_db', 0), redis_password=kwargs.get('redis_password'), redis_namespace=kwargs.get('redis_namespace', 'jormungandr.rate_limiter')) else: self.rate_limiter = FakeRateLimiter()
def __init__( self, id, service_url, service_args, timezone, object_id_tag=None, destination_id_tag=None, instance=None, timeout=10, **kwargs ): self.service_url = service_url self.service_args = service_args self.timeout = timeout # timeout in seconds self.rt_system_id = id self.object_id_tag = object_id_tag if object_id_tag else id self.destination_id_tag = destination_id_tag if destination_id_tag else "source" self.timeo_stop_code = kwargs.get("source_stop_code", "StopTimeoCode") self.timeo_line_code = kwargs.get("source_line_code", "LineTimeoCode") self.next_stop_time_number = kwargs.get("next_stop_time_number", 5) self.instance = instance fail_max = kwargs.get( 'circuit_breaker_max_fail', app.config.get(str('CIRCUIT_BREAKER_MAX_TIMEO_FAIL'), 5) ) reset_timeout = kwargs.get( 'circuit_breaker_reset_timeout', app.config.get(str('CIRCUIT_BREAKER_TIMEO_TIMEOUT_S'), 60) ) self.breaker = pybreaker.CircuitBreaker(fail_max=fail_max, reset_timeout=reset_timeout) # A step is applied on from_datetime to discretize calls and allow caching them self.from_datetime_step = kwargs.get( 'from_datetime_step', app.config.get(str('CACHE_CONFIGURATION'), {}).get(str('TIMEOUT_TIMEO'), 60) ) # Note: if the timezone is not know, pytz raise an error self.timezone = pytz.timezone(timezone) if kwargs.get('redis_host') and kwargs.get('rate_limit_count'): self.rate_limiter = RateLimiter( conditions=[ {'requests': kwargs.get('rate_limit_count'), 'seconds': kwargs.get('rate_limit_duration', 1)} ], redis_host=kwargs.get('redis_host'), redis_port=kwargs.get('redis_port', 6379), redis_db=kwargs.get('redis_db', 0), redis_password=kwargs.get('redis_password'), redis_namespace=kwargs.get('redis_namespace', 'jormungandr.rate_limiter'), ) else: self.rate_limiter = FakeRateLimiter() # We consider that all errors, greater than or equal to 100, are blocking self.INTERNAL_TIMEO_ERROR_CODE_LIMIT = 100
def __init__(self, id, service_url, timezone, object_id_tag=None, destination_id_tag=None, instance=None, timeout=10, redis_host=None, redis_db=0, redis_port=6379, redis_password=None, max_requests_by_second=15, redis_namespace='jormungandr.rate_limiter', **kwargs): self.service_url = service_url self.timeout = timeout # timeout in seconds self.rt_system_id = id self.object_id_tag = object_id_tag if object_id_tag else id self.destination_id_tag = destination_id_tag self.instance = instance fail_max = kwargs.get( 'circuit_breaker_max_fail', app.config.get(str('CIRCUIT_BREAKER_MAX_SYNTHESE_FAIL'), 5)) reset_timeout = kwargs.get( 'circuit_breaker_reset_timeout', app.config.get(str('CIRCUIT_BREAKER_SYNTHESE_TIMEOUT_S'), 60)) self.breaker = pybreaker.CircuitBreaker(fail_max=fail_max, reset_timeout=reset_timeout) self.timezone = pytz.timezone(timezone) if not redis_host: self.rate_limiter = FakeRateLimiter() else: self.rate_limiter = RateLimiter( conditions=[{ 'requests': max_requests_by_second, 'seconds': 1 }], redis_host=redis_host, redis_port=redis_port, redis_db=redis_db, redis_password=redis_password, redis_namespace=redis_namespace, )
class Timeo(RealtimeProxy): """ class managing calls to timeo external service providing real-time next passages """ def __init__(self, id, service_url, service_args, timezone, object_id_tag=None, destination_id_tag=None, instance=None, timeout=10, **kwargs): self.service_url = service_url self.service_args = service_args self.timeout = timeout #timeout in seconds self.rt_system_id = id self.object_id_tag = object_id_tag if object_id_tag else id self.destination_id_tag = destination_id_tag self.instance = instance self.breaker = pybreaker.CircuitBreaker(fail_max=app.config['CIRCUIT_BREAKER_MAX_TIMEO_FAIL'], reset_timeout=app.config['CIRCUIT_BREAKER_TIMEO_TIMEOUT_S']) # Note: if the timezone is not know, pytz raise an error self.timezone = pytz.timezone(timezone) if kwargs.get('redis_host') and kwargs.get('rate_limit_count'): self.rate_limiter = RateLimiter(conditions=[{'requests': kwargs.get('rate_limit_count'), 'seconds': kwargs.get('rate_limit_duration', 1)}], redis_host=kwargs.get('redis_host'), redis_port=kwargs.get('redis_port', 6379), redis_db=kwargs.get('redis_db', 0), redis_password=kwargs.get('redis_password'), redis_namespace=kwargs.get('redis_namespace', 'jormungandr.rate_limiter')) else: self.rate_limiter = FakeRateLimiter() def __repr__(self): """ used as the cache key. we use the rt_system_id to share the cache between servers in production """ return self.rt_system_id @cache.memoize(app.config['CACHE_CONFIGURATION'].get('TIMEOUT_TIMEO', 60)) def _call_timeo(self, url): """ http call to timeo The call is handled by a circuit breaker not to continue calling timeo if the service is dead. The call is also cached """ try: if not self.rate_limiter.acquire(self.rt_system_id, block=False): return None return self.breaker.call(requests.get, url, timeout=self.timeout) except pybreaker.CircuitBreakerError as e: logging.getLogger(__name__).error('Timeo RT service dead, using base ' 'schedule (error: {}'.format(e)) except requests.Timeout as t: logging.getLogger(__name__).error('Timeo RT service timeout, using base ' 'schedule (error: {}'.format(t)) except: logging.getLogger(__name__).exception('Timeo RT error, using base schedule') return None def _get_dt_local(self, utc_dt): return pytz.utc.localize(utc_dt).astimezone(self.timezone) def _is_tomorrow(self, request_dt, current_dt): if not request_dt: return False if not current_dt: now = self._get_dt_local(datetime.utcnow()) else: now = self._get_dt_local(current_dt) req_dt = self._timestamp_to_date(request_dt) return now.date() < req_dt.date() def _get_next_passage_for_route_point(self, route_point, count=None, from_dt=None, current_dt=None): if self._is_tomorrow(from_dt, current_dt): logging.getLogger(__name__).info('Timeo RT service , Can not call Timeo for tomorrow.') return None url = self._make_url(route_point, count, from_dt) if not url: return None logging.getLogger(__name__).debug('Timeo RT service , call url : {}'.format(url)) r = self._call_timeo(url) if not r: return None if r.status_code != 200: # TODO better error handling, the response might be in 200 but in error logging.getLogger(__name__).error('Timeo RT service unavailable, impossible to query : {}' .format(r.url)) return None return self._get_passages(r.json(), route_point.fetch_line_uri()) def _get_passages(self, timeo_resp, line_uri=None): logging.getLogger(__name__).debug('timeo response: {}'.format(timeo_resp)) st_responses = timeo_resp.get('StopTimesResponse') # by construction there should be only one StopTimesResponse if not st_responses or len(st_responses) != 1: logging.getLogger(__name__).warning('invalid timeo response: {}'.format(timeo_resp)) return None next_st = st_responses[0]['NextStopTimesMessage'] next_passages = [] for next_expected_st in next_st.get('NextExpectedStopTime', []): # for the moment we handle only the NextStop and the direction dt = self._get_dt(next_expected_st['NextStop']) direction = self._get_direction_name(line_uri=line_uri, object_code=next_expected_st.get('Terminus'), default_value=next_expected_st.get('Destination')) next_passage = RealTimePassage(dt, direction) next_passages.append(next_passage) return next_passages def _make_url(self, route_point, count=None, from_dt=None): """ the route point identifier is set with the StopDescription argument this argument is split in 3 arguments (given between '?' and ';' symbol....) * StopTimeoCode: timeo code for the stop * LineTimeoCode: timeo code for the line * Way: 'A' if the route is forward, 'R' if it is backward 2 additionals args are needed in this StopDescription ...: * NextStopTimeNumber: the number of next departure we want * StopTimeType: if we want base schedule data ('TH') or real time one ('TR') Note: since there are some strange symbol ('?' and ';') in the url we can't use param as dict in requests """ base_params = '&'.join([k + '=' + v for k, v in self.service_args.items()]) stop = route_point.fetch_stop_id(self.object_id_tag) line = route_point.fetch_line_id(self.object_id_tag) route = route_point.fetch_route_id(self.object_id_tag) if not all((stop, line, route)): # one a the id is missing, we'll not find any realtime logging.getLogger(__name__).debug('missing realtime id for {obj}: ' 'stop code={s}, line code={l}, route code={r}'. format(obj=route_point, s=stop, l=line, r=route)) return None # timeo can only handle items_per_schedule if it's < 5 count = min(count, 5) or 5 # if no value defined we ask for 5 passages # if a custom datetime is provided we give it to timeo dt_param = '&NextStopReferenceTime={dt}'\ .format(dt=self._timestamp_to_date(from_dt).strftime('%Y-%m-%dT%H:%M:%S')) \ if from_dt else '' stop_id_url = ("StopDescription=?" "StopTimeoCode={stop}" "&LineTimeoCode={line}" "&Way={route}" "&NextStopTimeNumber={count}" "&StopTimeType={data_freshness}{dt};").format(stop=stop, line=line, route=route, count=count, data_freshness='TR', dt=dt_param) url = "{base_url}?{base_params}&{stop_id}".format(base_url=self.service_url, base_params=base_params, stop_id=stop_id_url) return url def _get_dt(self, hour_str): hour = _to_duration(hour_str) # we then have to complete the hour with the date to have a datetime now = _get_current_date() dt = datetime.combine(now.date(), hour) utc_dt = self.timezone.normalize(self.timezone.localize(dt)).astimezone(pytz.utc) return utc_dt def _timestamp_to_date(self, timestamp): dt = datetime.utcfromtimestamp(timestamp) return self._get_dt_local(dt) def status(self): return {'id': self.rt_system_id, 'timeout': self.timeout, 'circuit_breaker': {'current_state': self.breaker.current_state, 'fail_counter': self.breaker.fail_counter, 'reset_timeout': self.breaker.reset_timeout}, } @cache.memoize(app.config['CACHE_CONFIGURATION'].get('TIMEOUT_PTOBJECTS', 600)) def _get_direction_name(self, line_uri, object_code, default_value): stop_point = self.instance.ptref.get_stop_point(line_uri, self.destination_id_tag, object_code) if stop_point and stop_point.HasField('stop_area') \ and stop_point.stop_area.HasField('label') \ and stop_point.stop_area.label != '': return stop_point.stop_area.label return default_value
class Timeo(RealtimeProxy): """ class managing calls to timeo external service providing real-time next passages curl example to check/test that external service is working: curl -X GET '{server}?serviceID={service}&EntityID={entity}&Media={spec}&StopDescription=?StopTimeoCode={stop_code}&NextStopTimeNumber={nb_desired}&LineTimeoCode={line_code}&Way={route_code}&StopTimeType=TR;' {line_code}, {route_code} and {stop_code} are provided using the same code key, named after the 'destination_id_tag' if provided on connector's init, or the 'id' otherwise. {route_code} is in pratice 'A' or 'R' (for 'aller' or 'retour', French for 'forth' or 'back'). So in practice it will look like: curl -X GET 'http://bobimeo.fr/cgi/api_compagnon.cgi?serviceID=9&EntityID=289&Media=spec_navit_comp&StopDescription=?StopTimeoCode=1586&NextStopTimeNumber=10&LineTimeoCode=52&Way=A&StopTimeType=TR;' """ def __init__(self, id, service_url, service_args, timezone, object_id_tag=None, destination_id_tag=None, instance=None, timeout=10, **kwargs): self.service_url = service_url self.service_args = service_args self.timeout = timeout # timeout in seconds self.rt_system_id = id self.object_id_tag = object_id_tag if object_id_tag else id self.destination_id_tag = destination_id_tag self.instance = instance fail_max = kwargs.get( 'circuit_breaker_max_fail', app.config.get(str('CIRCUIT_BREAKER_MAX_TIMEO_FAIL'), 5)) reset_timeout = kwargs.get( 'circuit_breaker_reset_timeout', app.config.get(str('CIRCUIT_BREAKER_TIMEO_TIMEOUT_S'), 60)) self.breaker = pybreaker.CircuitBreaker(fail_max=fail_max, reset_timeout=reset_timeout) # A step is applied on from_datetime to discretize calls and allow caching them self.from_datetime_step = kwargs.get( 'from_datetime_step', app.config.get(str('CACHE_CONFIGURATION'), {}).get(str('TIMEOUT_TIMEO'), 60)) # Note: if the timezone is not know, pytz raise an error self.timezone = pytz.timezone(timezone) if kwargs.get('redis_host') and kwargs.get('rate_limit_count'): self.rate_limiter = RateLimiter( conditions=[{ 'requests': kwargs.get('rate_limit_count'), 'seconds': kwargs.get('rate_limit_duration', 1) }], redis_host=kwargs.get('redis_host'), redis_port=kwargs.get('redis_port', 6379), redis_db=kwargs.get('redis_db', 0), redis_password=kwargs.get('redis_password'), redis_namespace=kwargs.get('redis_namespace', 'jormungandr.rate_limiter'), ) else: self.rate_limiter = FakeRateLimiter() # We consider that all errors, greater than or equal to 100, are blocking self.INTERNAL_TIMEO_ERROR_CODE_LIMIT = 100 def __repr__(self): """ used as the cache key. we use the rt_system_id to share the cache between servers in production """ try: return self.rt_system_id.encode('utf-8', 'backslashreplace') except: return self.rt_system_id @cache.memoize( app.config.get(str('CACHE_CONFIGURATION'), {}).get(str('TIMEOUT_TIMEO'), 60)) def _call_timeo(self, url): """ http call to timeo The call is handled by a circuit breaker not to continue calling timeo if the service is dead. The call is also cached """ try: if not self.rate_limiter.acquire(self.rt_system_id, block=False): return None return self.breaker.call(requests.get, url, timeout=self.timeout) except pybreaker.CircuitBreakerError as e: logging.getLogger(__name__).error( 'Timeo RT service dead, using base schedule (error: {}'.format( e), extra={'rt_system_id': six.text_type(self.rt_system_id)}, ) raise RealtimeProxyError('circuit breaker open') except requests.Timeout as t: logging.getLogger(__name__).error( 'Timeo RT service timeout, using base schedule (error: {}'. format(t), extra={'rt_system_id': six.text_type(self.rt_system_id)}, ) raise RealtimeProxyError('timeout') except Exception as e: logging.getLogger(__name__).exception( 'Timeo RT error, using base schedule', extra={'rt_system_id': six.text_type(self.rt_system_id)}) raise RealtimeProxyError(str(e)) def _get_dt_local(self, utc_dt): return pytz.utc.localize(utc_dt).astimezone(self.timezone) def _is_tomorrow(self, request_dt, current_dt): if not request_dt: return False if not current_dt: now = self._get_dt_local(datetime.utcnow()) else: now = self._get_dt_local(current_dt) req_dt = self._timestamp_to_date(request_dt) return now.date() < req_dt.date() def _get_next_passage_for_route_point(self, route_point, count=None, from_dt=None, current_dt=None, duration=None): if self._is_tomorrow(from_dt, current_dt): logging.getLogger(__name__).info( 'Timeo RT service , Can not call Timeo for tomorrow.', extra={'rt_system_id': six.text_type(self.rt_system_id)}, ) return None url = self._make_url(route_point, count, from_dt) if not url: return None logging.getLogger(__name__).debug( 'Timeo RT service , call url : {}'.format(url), extra={'rt_system_id': six.text_type(self.rt_system_id)}, ) r = self._call_timeo(url) if not r: return None return self._get_passages(r, current_dt, route_point.fetch_line_uri()) def _get_passages(self, response, current_dt, line_uri=None): status_code = response.status_code timeo_resp = response.json() logging.getLogger(__name__).debug( 'timeo response: {}'.format(timeo_resp), extra={'rt_system_id': six.text_type(self.rt_system_id)}) # Handling http error if status_code != 200: logging.getLogger(__name__).error( 'Timeo RT service unavailable, impossible to query : {}'. format(response.url), extra={ 'rt_system_id': six.text_type(self.rt_system_id), 'status_code': status_code }, ) raise RealtimeProxyError('non 200 response') # internal timeo error handling message_responses = timeo_resp.get('MessageResponse') for message_response in message_responses: if ('ResponseCode' in message_response and int(message_response['ResponseCode']) >= self.INTERNAL_TIMEO_ERROR_CODE_LIMIT): resp_code = message_response['ResponseCode'] if 'ResponseComment' in message_response: resp_comment = message_response['ResponseComment'] else: resp_comment = '' self.record_internal_failure( 'Timeo RT internal service error', 'ResponseCode: {} - ResponseComment: {}'.format( resp_code, resp_comment), ) timeo_internal_error_message = 'Timeo RT internal service error, ResponseCode: {} - ResponseComment: {}'.format( resp_code, resp_comment) logging.getLogger(__name__).error(timeo_internal_error_message) raise RealtimeProxyError(timeo_internal_error_message) st_responses = timeo_resp.get('StopTimesResponse') # by construction there should be only one StopTimesResponse if not st_responses or len(st_responses) != 1: logging.getLogger(__name__).warning( 'invalid timeo response: {}'.format(timeo_resp), extra={'rt_system_id': six.text_type(self.rt_system_id)}, ) raise RealtimeProxyError('invalid response') next_st = st_responses[0]['NextStopTimesMessage'] next_passages = [] for next_expected_st in next_st.get('NextExpectedStopTime', []): # for the moment we handle only the NextStop and the direction dt = self._get_dt(next_expected_st['NextStop'], current_dt) direction = self._get_direction_name( line_uri=line_uri, object_code=next_expected_st.get('Terminus'), default_value=next_expected_st.get('Destination'), ) next_passage = RealTimePassage(dt, direction) next_passages.append(next_passage) return next_passages def _make_url(self, route_point, count=None, from_dt=None): """ the route point identifier is set with the StopDescription argument this argument is split in 3 arguments (given between '?' and ';' symbol....) * StopTimeoCode: timeo code for the stop * LineTimeoCode: timeo code for the line * Way: 'A' if the route is forward, 'R' if it is backward 2 additionals args are needed in this StopDescription ...: * NextStopTimeNumber: the number of next departure we want * StopTimeType: if we want base schedule data ('TH') or real time one ('TR') Note: since there are some strange symbol ('?' and ';') in the url we can't use param as dict in requests """ base_params = '&'.join( [k + '=' + v for k, v in self.service_args.items()]) stop = route_point.fetch_stop_id(self.object_id_tag) line = route_point.fetch_line_id(self.object_id_tag) route = route_point.fetch_route_id(self.object_id_tag) if not all((stop, line, route)): # one a the id is missing, we'll not find any realtime logging.getLogger(__name__).debug( 'missing realtime id for {obj}: ' 'stop code={s}, line code={l}, route code={r}'.format( obj=route_point, s=stop, l=line, r=route), extra={'rt_system_id': six.text_type(self.rt_system_id)}, ) self.record_internal_failure('missing id') return None # timeo can only handle items_per_schedule if it's < 5 count = min(count or 5, 5) # if no value defined we ask for 5 passages # if a custom datetime is provided we give it to timeo but we round it to improve cachability dt_param = ('&NextStopReferenceTime={dt}'.format(dt=floor_datetime( self._timestamp_to_date(from_dt), self.from_datetime_step ).strftime('%Y-%m-%dT%H:%M:%S')) if from_dt else '') # We want to have StopTimeType as it make parsing of the request way easier # for alternative implementation of timeo since we can ignore this params stop_id_url = ("StopDescription=?" "StopTimeType={data_freshness}" "&LineTimeoCode={line}" "&Way={route}" "&NextStopTimeNumber={count}" "&StopTimeoCode={stop}{dt};").format( stop=stop, line=line, route=route, count=count, data_freshness='TR', dt=dt_param) url = "{base_url}?{base_params}&{stop_id}".format( base_url=self.service_url, base_params=base_params, stop_id=stop_id_url) return url def _get_dt(self, hour_str, current_dt): hour = _to_duration(hour_str) # we then have to complete the hour with the date to have a datetime now = current_dt dt = datetime.combine(now.date(), hour) utc_dt = self.timezone.normalize( self.timezone.localize(dt)).astimezone(pytz.utc) return utc_dt def _timestamp_to_date(self, timestamp): dt = datetime.utcfromtimestamp(timestamp) return self._get_dt_local(dt) def status(self): return { 'id': six.text_type(self.rt_system_id), 'timeout': self.timeout, 'circuit_breaker': { 'current_state': self.breaker.current_state, 'fail_counter': self.breaker.fail_counter, 'reset_timeout': self.breaker.reset_timeout, }, } @cache.memoize( app.config.get(str('CACHE_CONFIGURATION'), {}).get(str('TIMEOUT_PTOBJECTS'), 600)) def _get_direction_name(self, line_uri, object_code, default_value): stop_point = self.instance.ptref.get_stop_point( line_uri, self.destination_id_tag, object_code) if (stop_point and stop_point.HasField('stop_area') and stop_point.stop_area.HasField('label') and stop_point.stop_area.label != ''): return stop_point.stop_area.label return default_value