class Timeo(RealtimeProxy):
"""
class managing calls to timeo external service providing real-time next passages
"""
def __init__(self, id, service_url, service_args, timezone,
object_id_tag=None, destination_id_tag=None, instance=None, timeout=10, **kwargs):
self.service_url = service_url
self.service_args = service_args
self.timeout = timeout #timeout in seconds
self.rt_system_id = id
self.object_id_tag = object_id_tag if object_id_tag else id
self.destination_id_tag = destination_id_tag
self.instance = instance
self.breaker = pybreaker.CircuitBreaker(fail_max=app.config['CIRCUIT_BREAKER_MAX_TIMEO_FAIL'],
reset_timeout=app.config['CIRCUIT_BREAKER_TIMEO_TIMEOUT_S'])
# Note: if the timezone is not know, pytz raise an error
self.timezone = pytz.timezone(timezone)
if kwargs.get('redis_host') and kwargs.get('rate_limit_count'):
self.rate_limiter = RateLimiter(conditions=[{'requests': kwargs.get('rate_limit_count'),
'seconds': kwargs.get('rate_limit_duration', 1)}],
redis_host=kwargs.get('redis_host'),
redis_port=kwargs.get('redis_port', 6379),
redis_db=kwargs.get('redis_db', 0),
redis_password=kwargs.get('redis_password'),
redis_namespace=kwargs.get('redis_namespace', 'jormungandr.rate_limiter'))
else:
self.rate_limiter = FakeRateLimiter()
def __repr__(self):
"""
used as the cache key. we use the rt_system_id to share the cache between servers in production
"""
return self.rt_system_id
@cache.memoize(app.config['CACHE_CONFIGURATION'].get('TIMEOUT_TIMEO', 60))
def _call_timeo(self, url):
"""
http call to timeo
The call is handled by a circuit breaker not to continue calling timeo if the service is dead.
The call is also cached
"""
try:
if not self.rate_limiter.acquire(self.rt_system_id, block=False):
return None
return self.breaker.call(requests.get, url, timeout=self.timeout)
except pybreaker.CircuitBreakerError as e:
logging.getLogger(__name__).error('Timeo RT service dead, using base '
'schedule (error: {}'.format(e))
except requests.Timeout as t:
logging.getLogger(__name__).error('Timeo RT service timeout, using base '
'schedule (error: {}'.format(t))
except:
logging.getLogger(__name__).exception('Timeo RT error, using base schedule')
return None
def _get_dt_local(self, utc_dt):
return pytz.utc.localize(utc_dt).astimezone(self.timezone)
def _is_tomorrow(self, request_dt, current_dt):
if not request_dt:
return False
if not current_dt:
now = self._get_dt_local(datetime.utcnow())
else:
now = self._get_dt_local(current_dt)
req_dt = self._timestamp_to_date(request_dt)
return now.date() < req_dt.date()
def _get_next_passage_for_route_point(self, route_point, count=None, from_dt=None, current_dt=None):
if self._is_tomorrow(from_dt, current_dt):
logging.getLogger(__name__).info('Timeo RT service , Can not call Timeo for tomorrow.')
return None
url = self._make_url(route_point, count, from_dt)
if not url:
return None
logging.getLogger(__name__).debug('Timeo RT service , call url : {}'.format(url))
r = self._call_timeo(url)
if not r:
return None
if r.status_code != 200:
# TODO better error handling, the response might be in 200 but in error
logging.getLogger(__name__).error('Timeo RT service unavailable, impossible to query : {}'
.format(r.url))
return None
return self._get_passages(r.json(), route_point.fetch_line_uri())
def _get_passages(self, timeo_resp, line_uri=None):
logging.getLogger(__name__).debug('timeo response: {}'.format(timeo_resp))
st_responses = timeo_resp.get('StopTimesResponse')
# by construction there should be only one StopTimesResponse
if not st_responses or len(st_responses) != 1:
logging.getLogger(__name__).warning('invalid timeo response: {}'.format(timeo_resp))
return None
next_st = st_responses[0]['NextStopTimesMessage']
next_passages = []
for next_expected_st in next_st.get('NextExpectedStopTime', []):
# for the moment we handle only the NextStop and the direction
dt = self._get_dt(next_expected_st['NextStop'])
direction = self._get_direction_name(line_uri=line_uri,
object_code=next_expected_st.get('Terminus'),
default_value=next_expected_st.get('Destination'))
next_passage = RealTimePassage(dt, direction)
next_passages.append(next_passage)
return next_passages
def _make_url(self, route_point, count=None, from_dt=None):
"""
the route point identifier is set with the StopDescription argument
this argument is split in 3 arguments (given between '?' and ';' symbol....)
* StopTimeoCode: timeo code for the stop
* LineTimeoCode: timeo code for the line
* Way: 'A' if the route is forward, 'R' if it is backward
2 additionals args are needed in this StopDescription ...:
* NextStopTimeNumber: the number of next departure we want
* StopTimeType: if we want base schedule data ('TH') or real time one ('TR')
Note: since there are some strange symbol ('?' and ';') in the url we can't use param as dict in
requests
"""
base_params = '&'.join([k + '=' + v for k, v in self.service_args.items()])
stop = route_point.fetch_stop_id(self.object_id_tag)
line = route_point.fetch_line_id(self.object_id_tag)
route = route_point.fetch_route_id(self.object_id_tag)
if not all((stop, line, route)):
# one a the id is missing, we'll not find any realtime
logging.getLogger(__name__).debug('missing realtime id for {obj}: '
'stop code={s}, line code={l}, route code={r}'.
format(obj=route_point, s=stop, l=line, r=route))
return None
# timeo can only handle items_per_schedule if it's < 5
count = min(count, 5) or 5 # if no value defined we ask for 5 passages
# if a custom datetime is provided we give it to timeo
dt_param = '&NextStopReferenceTime={dt}'\
.format(dt=self._timestamp_to_date(from_dt).strftime('%Y-%m-%dT%H:%M:%S')) \
if from_dt else ''
stop_id_url = ("StopDescription=?"
"StopTimeoCode={stop}"
"&LineTimeoCode={line}"
"&Way={route}"
"&NextStopTimeNumber={count}"
"&StopTimeType={data_freshness}{dt};").format(stop=stop,
line=line,
route=route,
count=count,
data_freshness='TR',
dt=dt_param)
url = "{base_url}?{base_params}&{stop_id}".format(base_url=self.service_url,
base_params=base_params,
stop_id=stop_id_url)
return url
def _get_dt(self, hour_str):
hour = _to_duration(hour_str)
# we then have to complete the hour with the date to have a datetime
now = _get_current_date()
dt = datetime.combine(now.date(), hour)
utc_dt = self.timezone.normalize(self.timezone.localize(dt)).astimezone(pytz.utc)
return utc_dt
def _timestamp_to_date(self, timestamp):
dt = datetime.utcfromtimestamp(timestamp)
return self._get_dt_local(dt)
def status(self):
return {'id': self.rt_system_id,
'timeout': self.timeout,
'circuit_breaker': {'current_state': self.breaker.current_state,
'fail_counter': self.breaker.fail_counter,
'reset_timeout': self.breaker.reset_timeout},
}
@cache.memoize(app.config['CACHE_CONFIGURATION'].get('TIMEOUT_PTOBJECTS', 600))
def _get_direction_name(self, line_uri, object_code, default_value):
stop_point = self.instance.ptref.get_stop_point(line_uri, self.destination_id_tag, object_code)
if stop_point and stop_point.HasField('stop_area') \
and stop_point.stop_area.HasField('label') \
and stop_point.stop_area.label != '':
return stop_point.stop_area.label
return default_value
class Timeo(RealtimeProxy):
"""
class managing calls to timeo external service providing real-time next passages
curl example to check/test that external service is working:
curl -X GET '{server}?serviceID={service}&EntityID={entity}&Media={spec}&StopDescription=?StopTimeoCode={stop_code}&NextStopTimeNumber={nb_desired}&LineTimeoCode={line_code}&Way={route_code}&StopTimeType=TR;'
{line_code}, {route_code} and {stop_code} are provided using the same code key, named after
the 'destination_id_tag' if provided on connector's init, or the 'id' otherwise.
{route_code} is in pratice 'A' or 'R' (for 'aller' or 'retour', French for 'forth' or 'back').
So in practice it will look like:
curl -X GET 'http://bobimeo.fr/cgi/api_compagnon.cgi?serviceID=9&EntityID=289&Media=spec_navit_comp&StopDescription=?StopTimeoCode=1586&NextStopTimeNumber=10&LineTimeoCode=52&Way=A&StopTimeType=TR;'
"""
def __init__(self,
id,
service_url,
service_args,
timezone,
object_id_tag=None,
destination_id_tag=None,
instance=None,
timeout=10,
**kwargs):
self.service_url = service_url
self.service_args = service_args
self.timeout = timeout # timeout in seconds
self.rt_system_id = id
self.object_id_tag = object_id_tag if object_id_tag else id
self.destination_id_tag = destination_id_tag
self.instance = instance
fail_max = kwargs.get(
'circuit_breaker_max_fail',
app.config.get(str('CIRCUIT_BREAKER_MAX_TIMEO_FAIL'), 5))
reset_timeout = kwargs.get(
'circuit_breaker_reset_timeout',
app.config.get(str('CIRCUIT_BREAKER_TIMEO_TIMEOUT_S'), 60))
self.breaker = pybreaker.CircuitBreaker(fail_max=fail_max,
reset_timeout=reset_timeout)
# A step is applied on from_datetime to discretize calls and allow caching them
self.from_datetime_step = kwargs.get(
'from_datetime_step',
app.config.get(str('CACHE_CONFIGURATION'),
{}).get(str('TIMEOUT_TIMEO'), 60))
# Note: if the timezone is not know, pytz raise an error
self.timezone = pytz.timezone(timezone)
if kwargs.get('redis_host') and kwargs.get('rate_limit_count'):
self.rate_limiter = RateLimiter(
conditions=[{
'requests': kwargs.get('rate_limit_count'),
'seconds': kwargs.get('rate_limit_duration', 1)
}],
redis_host=kwargs.get('redis_host'),
redis_port=kwargs.get('redis_port', 6379),
redis_db=kwargs.get('redis_db', 0),
redis_password=kwargs.get('redis_password'),
redis_namespace=kwargs.get('redis_namespace',
'jormungandr.rate_limiter'),
)
else:
self.rate_limiter = FakeRateLimiter()
# We consider that all errors, greater than or equal to 100, are blocking
self.INTERNAL_TIMEO_ERROR_CODE_LIMIT = 100
def __repr__(self):
"""
used as the cache key. we use the rt_system_id to share the cache between servers in production
"""
try:
return self.rt_system_id.encode('utf-8', 'backslashreplace')
except:
return self.rt_system_id
@cache.memoize(
app.config.get(str('CACHE_CONFIGURATION'),
{}).get(str('TIMEOUT_TIMEO'), 60))
def _call_timeo(self, url):
"""
http call to timeo
The call is handled by a circuit breaker not to continue calling timeo if the service is dead.
The call is also cached
"""
try:
if not self.rate_limiter.acquire(self.rt_system_id, block=False):
return None
return self.breaker.call(requests.get, url, timeout=self.timeout)
except pybreaker.CircuitBreakerError as e:
logging.getLogger(__name__).error(
'Timeo RT service dead, using base schedule (error: {}'.format(
e),
extra={'rt_system_id': six.text_type(self.rt_system_id)},
)
raise RealtimeProxyError('circuit breaker open')
except requests.Timeout as t:
logging.getLogger(__name__).error(
'Timeo RT service timeout, using base schedule (error: {}'.
format(t),
extra={'rt_system_id': six.text_type(self.rt_system_id)},
)
raise RealtimeProxyError('timeout')
except Exception as e:
logging.getLogger(__name__).exception(
'Timeo RT error, using base schedule',
extra={'rt_system_id': six.text_type(self.rt_system_id)})
raise RealtimeProxyError(str(e))
def _get_dt_local(self, utc_dt):
return pytz.utc.localize(utc_dt).astimezone(self.timezone)
def _is_tomorrow(self, request_dt, current_dt):
if not request_dt:
return False
if not current_dt:
now = self._get_dt_local(datetime.utcnow())
else:
now = self._get_dt_local(current_dt)
req_dt = self._timestamp_to_date(request_dt)
return now.date() < req_dt.date()
def _get_next_passage_for_route_point(self,
route_point,
count=None,
from_dt=None,
current_dt=None,
duration=None):
if self._is_tomorrow(from_dt, current_dt):
logging.getLogger(__name__).info(
'Timeo RT service , Can not call Timeo for tomorrow.',
extra={'rt_system_id': six.text_type(self.rt_system_id)},
)
return None
url = self._make_url(route_point, count, from_dt)
if not url:
return None
logging.getLogger(__name__).debug(
'Timeo RT service , call url : {}'.format(url),
extra={'rt_system_id': six.text_type(self.rt_system_id)},
)
r = self._call_timeo(url)
if not r:
return None
return self._get_passages(r, current_dt, route_point.fetch_line_uri())
def _get_passages(self, response, current_dt, line_uri=None):
status_code = response.status_code
timeo_resp = response.json()
logging.getLogger(__name__).debug(
'timeo response: {}'.format(timeo_resp),
extra={'rt_system_id': six.text_type(self.rt_system_id)})
# Handling http error
if status_code != 200:
logging.getLogger(__name__).error(
'Timeo RT service unavailable, impossible to query : {}'.
format(response.url),
extra={
'rt_system_id': six.text_type(self.rt_system_id),
'status_code': status_code
},
)
raise RealtimeProxyError('non 200 response')
# internal timeo error handling
message_responses = timeo_resp.get('MessageResponse')
for message_response in message_responses:
if ('ResponseCode' in message_response
and int(message_response['ResponseCode']) >=
self.INTERNAL_TIMEO_ERROR_CODE_LIMIT):
resp_code = message_response['ResponseCode']
if 'ResponseComment' in message_response:
resp_comment = message_response['ResponseComment']
else:
resp_comment = ''
self.record_internal_failure(
'Timeo RT internal service error',
'ResponseCode: {} - ResponseComment: {}'.format(
resp_code, resp_comment),
)
timeo_internal_error_message = 'Timeo RT internal service error, ResponseCode: {} - ResponseComment: {}'.format(
resp_code, resp_comment)
logging.getLogger(__name__).error(timeo_internal_error_message)
raise RealtimeProxyError(timeo_internal_error_message)
st_responses = timeo_resp.get('StopTimesResponse')
# by construction there should be only one StopTimesResponse
if not st_responses or len(st_responses) != 1:
logging.getLogger(__name__).warning(
'invalid timeo response: {}'.format(timeo_resp),
extra={'rt_system_id': six.text_type(self.rt_system_id)},
)
raise RealtimeProxyError('invalid response')
next_st = st_responses[0]['NextStopTimesMessage']
next_passages = []
for next_expected_st in next_st.get('NextExpectedStopTime', []):
# for the moment we handle only the NextStop and the direction
dt = self._get_dt(next_expected_st['NextStop'], current_dt)
direction = self._get_direction_name(
line_uri=line_uri,
object_code=next_expected_st.get('Terminus'),
default_value=next_expected_st.get('Destination'),
)
next_passage = RealTimePassage(dt, direction)
next_passages.append(next_passage)
return next_passages
def _make_url(self, route_point, count=None, from_dt=None):
"""
the route point identifier is set with the StopDescription argument
this argument is split in 3 arguments (given between '?' and ';' symbol....)
* StopTimeoCode: timeo code for the stop
* LineTimeoCode: timeo code for the line
* Way: 'A' if the route is forward, 'R' if it is backward
2 additionals args are needed in this StopDescription ...:
* NextStopTimeNumber: the number of next departure we want
* StopTimeType: if we want base schedule data ('TH') or real time one ('TR')
Note: since there are some strange symbol ('?' and ';') in the url we can't use param as dict in
requests
"""
base_params = '&'.join(
[k + '=' + v for k, v in self.service_args.items()])
stop = route_point.fetch_stop_id(self.object_id_tag)
line = route_point.fetch_line_id(self.object_id_tag)
route = route_point.fetch_route_id(self.object_id_tag)
if not all((stop, line, route)):
# one a the id is missing, we'll not find any realtime
logging.getLogger(__name__).debug(
'missing realtime id for {obj}: '
'stop code={s}, line code={l}, route code={r}'.format(
obj=route_point, s=stop, l=line, r=route),
extra={'rt_system_id': six.text_type(self.rt_system_id)},
)
self.record_internal_failure('missing id')
return None
# timeo can only handle items_per_schedule if it's < 5
count = min(count or 5, 5) # if no value defined we ask for 5 passages
# if a custom datetime is provided we give it to timeo but we round it to improve cachability
dt_param = ('&NextStopReferenceTime={dt}'.format(dt=floor_datetime(
self._timestamp_to_date(from_dt), self.from_datetime_step
).strftime('%Y-%m-%dT%H:%M:%S')) if from_dt else '')
# We want to have StopTimeType as it make parsing of the request way easier
# for alternative implementation of timeo since we can ignore this params
stop_id_url = ("StopDescription=?"
"StopTimeType={data_freshness}"
"&LineTimeoCode={line}"
"&Way={route}"
"&NextStopTimeNumber={count}"
"&StopTimeoCode={stop}{dt};").format(
stop=stop,
line=line,
route=route,
count=count,
data_freshness='TR',
dt=dt_param)
url = "{base_url}?{base_params}&{stop_id}".format(
base_url=self.service_url,
base_params=base_params,
stop_id=stop_id_url)
return url
def _get_dt(self, hour_str, current_dt):
hour = _to_duration(hour_str)
# we then have to complete the hour with the date to have a datetime
now = current_dt
dt = datetime.combine(now.date(), hour)
utc_dt = self.timezone.normalize(
self.timezone.localize(dt)).astimezone(pytz.utc)
return utc_dt
def _timestamp_to_date(self, timestamp):
dt = datetime.utcfromtimestamp(timestamp)
return self._get_dt_local(dt)
def status(self):
return {
'id': six.text_type(self.rt_system_id),
'timeout': self.timeout,
'circuit_breaker': {
'current_state': self.breaker.current_state,
'fail_counter': self.breaker.fail_counter,
'reset_timeout': self.breaker.reset_timeout,
},
}
@cache.memoize(
app.config.get(str('CACHE_CONFIGURATION'),
{}).get(str('TIMEOUT_PTOBJECTS'), 600))
def _get_direction_name(self, line_uri, object_code, default_value):
stop_point = self.instance.ptref.get_stop_point(
line_uri, self.destination_id_tag, object_code)
if (stop_point and stop_point.HasField('stop_area')
and stop_point.stop_area.HasField('label')
and stop_point.stop_area.label != ''):
return stop_point.stop_area.label
return default_value