def _call_siri(self, request):
encoded_request = request.encode('utf-8', 'backslashreplace')
headers = {
"Content-Type": "text/xml; charset=UTF-8",
"Content-Length": str(len(encoded_request))
}
logging.getLogger(__name__).debug(
'siri RT service, post at {}: {}'.format(self.service_url,
request))
try:
return self.breaker.call(
requests.post,
url=self.service_url,
headers=headers,
data=encoded_request,
verify=False,
timeout=self.timeout,
)
except pybreaker.CircuitBreakerError as e:
logging.getLogger(__name__).error(
'siri RT service dead, using base '
'schedule (error: {}'.format(e))
raise RealtimeProxyError('circuit breaker open')
except requests.Timeout as t:
logging.getLogger(__name__).error(
'siri RT service timeout, using base '
'schedule (error: {}'.format(t))
raise RealtimeProxyError('timeout')
except Exception as e:
logging.getLogger(__name__).exception(
'siri RT error, using base schedule')
raise RealtimeProxyError(str(e))
def _get_next_passage_for_route_point(self,
route_point,
count,
from_dt,
current_dt,
duration=None):
stop = route_point.fetch_stop_id(self.object_id_tag)
request = self._make_request(monitoring_ref=stop,
dt=from_dt,
count=count)
if not request:
return None
siri_response = self._call_siri(request)
if not siri_response or siri_response.status_code != 200:
raise RealtimeProxyError('invalid response')
logging.getLogger(__name__).debug('siri for {}: {}'.format(
stop, siri_response.text))
ns = {'siri': 'http://www.siri.org.uk/siri'}
tree = None
try:
tree = et.fromstring(siri_response.content)
except et.ParseError:
logging.getLogger(__name__).exception("invalid xml")
raise RealtimeProxyError('invalid xml')
self._validate_response_or_raise(tree, ns)
return self._get_passages(tree, ns, route_point)
def _call_cleverage(self, url):
"""
http call to cleverage
"""
logging.getLogger(__name__).debug(
'Cleverage RT service , call url : {}'.format(url))
try:
return self.breaker.call(requests.get,
url,
timeout=self.timeout,
headers=self.service_args)
except pybreaker.CircuitBreakerError as e:
logging.getLogger(__name__).error(
'Cleverage RT service dead, using base '
'schedule (error: {}'.format(e))
raise RealtimeProxyError('circuit breaker open')
except requests.Timeout as t:
logging.getLogger(__name__).error(
'Cleverage RT service timeout, using base '
'schedule (error: {}'.format(t))
raise RealtimeProxyError('timeout')
except Exception as e:
logging.getLogger(__name__).exception(
'Cleverage RT error, using base schedule')
raise RealtimeProxyError(str(e))
def _call_timeo(self, url):
"""
http call to timeo
The call is handled by a circuit breaker not to continue calling timeo if the service is dead.
The call is also cached
"""
try:
if not self.rate_limiter.acquire(self.rt_system_id, block=False):
return None
return self.breaker.call(requests.get, url, timeout=self.timeout)
except pybreaker.CircuitBreakerError as e:
logging.getLogger(__name__).error(
'Timeo RT service dead, using base schedule (error: {}'.format(
e),
extra={'rt_system_id': six.text_type(self.rt_system_id)},
)
raise RealtimeProxyError('circuit breaker open')
except requests.Timeout as t:
logging.getLogger(__name__).error(
'Timeo RT service timeout, using base schedule (error: {}'.
format(t),
extra={'rt_system_id': six.text_type(self.rt_system_id)},
)
raise RealtimeProxyError('timeout')
except Exception as e:
logging.getLogger(__name__).exception(
'Timeo RT error, using base schedule',
extra={'rt_system_id': six.text_type(self.rt_system_id)})
raise RealtimeProxyError(str(e))
def _call_synthese(self, url):
"""
http call to synthese
"""
try:
if not self.rate_limiter.acquire(self.rt_system_id, block=False):
raise RealtimeProxyError('maximum rate reached')
return self.breaker.call(requests.get, url, timeout=self.timeout)
except pybreaker.CircuitBreakerError as e:
logging.getLogger(__name__).error(
'Synthese RT service dead, using base '
'schedule (error: {}'.format(e))
raise RealtimeProxyError('circuit breaker open')
except requests.Timeout as t:
logging.getLogger(__name__).error(
'Synthese RT service timeout, using base '
'schedule (error: {}'.format(t))
raise RealtimeProxyError('timeout')
except redis.ConnectionError:
logging.getLogger(__name__).exception(
'there is an error with Redis')
raise RealtimeProxyError('redis error')
except Exception as e:
logging.getLogger(__name__).exception(
'Synthese RT error, using base schedule')
raise RealtimeProxyError(str(e))
def _validate_response_or_raise(self, tree, ns):
stop_monitoring_delivery = tree.find('.//siri:StopMonitoringDelivery', ns)
if stop_monitoring_delivery is None:
raise RealtimeProxyError('No StopMonitoringDelivery in response')
status = stop_monitoring_delivery.find('.//siri:Status', ns)
if status is not None and not to_bool(status.text):
# Status is false: there is a problem, but we may have a valid response too...
# Lets log whats happening
error_condition = stop_monitoring_delivery.find('.//siri:ErrorCondition', ns)
if error_condition is not None and list(error_condition):
if error_condition.find('.//siri:NoInfoForTopicError', ns) is not None:
# There is no data, we might be at the end of the service
# OR the SIRI server doesn't update it's own data: there is no way to know
# let's say it's normal and not log nor return base_schedule data
return
# Log the error returned by SIRI, there is a node for the normalized error code
# and another node that holds the description
code = " ".join([e.tag for e in list(error_condition) if 'Description' not in e.tag])
description_node = error_condition.find('.//siri:Description', ns)
description = description_node.text if description_node is not None else None
logging.getLogger(__name__).warning('error in siri response: %s/%s', code, description)
monitored_stops = stop_monitoring_delivery.findall('.//siri:MonitoredStopVisit', ns)
if monitored_stops is None or len(monitored_stops) < 1:
# we might want to ignore error that match siri:NoInfoForTopicError,
# maybe it means that there is no next departure, maybe not...
# There is no departures and status is false: this looks like a real error...
# If description contains error message use it in exception (ex:[BAD_ID] MonitoringRef (01001713:TOC))
message = description or 'response status = false'
raise RealtimeProxyError(message)
def _call(self, url):
"""
http call to sytralRT
"""
logging.getLogger(__name__).debug(
'systralRT RT service , call url : {}'.format(url),
extra={'rt_system_id': unicode(self.rt_system_id)},
)
try:
return self.breaker.call(requests.get, url, timeout=self.timeout)
except pybreaker.CircuitBreakerError as e:
logging.getLogger(__name__).error(
'systralRT service dead, using base '
'schedule (error: {}'.format(e),
extra={'rt_system_id': unicode(self.rt_system_id)},
)
raise RealtimeProxyError('circuit breaker open')
except requests.Timeout as t:
logging.getLogger(__name__).error(
'systralRT service timeout, using base '
'schedule (error: {}'.format(t),
extra={'rt_system_id': unicode(self.rt_system_id)},
)
raise RealtimeProxyError('timeout')
except Exception as e:
logging.getLogger(__name__).exception(
'systralRT RT error, using base schedule',
extra={'rt_system_id': unicode(self.rt_system_id)})
raise RealtimeProxyError(str(e))
def _get_next_passage_for_route_point(self,
route_point,
count=None,
from_dt=None,
current_dt=None,
duration=None):
if self._is_tomorrow(from_dt, current_dt):
logging.getLogger(__name__).info(
'Timeo RT service , Can not call Timeo for tomorrow.',
extra={'rt_system_id': unicode(self.rt_system_id)})
return None
url = self._make_url(route_point, count, from_dt)
if not url:
return None
logging.getLogger(__name__).debug(
'Timeo RT service , call url : {}'.format(url),
extra={'rt_system_id': unicode(self.rt_system_id)})
r = self._call_timeo(url)
if not r:
return None
if r.status_code != 200:
# TODO better error handling, the response might be in 200 but in error
logging.getLogger(__name__).error(
'Timeo RT service unavailable, impossible to query : {}'.
format(r.url),
extra={
'rt_system_id': unicode(self.rt_system_id),
'status_code': r.status_code
})
raise RealtimeProxyError('non 200 response')
return self._get_passages(r.json(), current_dt,
route_point.fetch_line_uri())
def _get_passages(self, xml, route_point):
ns = {'siri': 'http://www.siri.org.uk/siri'}
try:
root = et.fromstring(xml)
except et.ParseError as e:
logging.getLogger(__name__).exception("invalid xml")
raise RealtimeProxyError('invalid xml')
stop = route_point.fetch_stop_id(self.object_id_tag)
line = route_point.fetch_line_id(self.object_id_tag)
route = route_point.fetch_route_id(self.object_id_tag)
next_passages = []
for visit in root.findall('.//siri:MonitoredStopVisit', ns):
cur_stop = visit.find('.//siri:StopPointRef', ns).text
if stop != cur_stop:
continue
cur_line = visit.find('.//siri:LineRef', ns).text
if line != cur_line:
continue
cur_route = visit.find('.//siri:DirectionName', ns).text
if route != cur_route:
continue
cur_destination = visit.find('.//siri:DestinationName', ns).text
cur_dt = visit.find('.//siri:ExpectedDepartureTime', ns).text
cur_dt = aniso8601.parse_datetime(cur_dt)
next_passages.append(RealTimePassage(cur_dt, cur_destination))
return next_passages
def _get_passages(self, timeo_resp, current_dt, line_uri=None):
logging.getLogger(__name__).debug(
'timeo response: {}'.format(timeo_resp),
extra={'rt_system_id': unicode(self.rt_system_id)})
st_responses = timeo_resp.get('StopTimesResponse')
# by construction there should be only one StopTimesResponse
if not st_responses or len(st_responses) != 1:
logging.getLogger(__name__).warning(
'invalid timeo response: {}'.format(timeo_resp),
extra={'rt_system_id': unicode(self.rt_system_id)})
raise RealtimeProxyError('invalid response')
next_st = st_responses[0]['NextStopTimesMessage']
next_passages = []
for next_expected_st in next_st.get('NextExpectedStopTime', []):
# for the moment we handle only the NextStop and the direction
dt = self._get_dt(next_expected_st['NextStop'], current_dt)
direction = self._get_direction_name(
line_uri=line_uri,
object_code=next_expected_st.get('Terminus'),
default_value=next_expected_st.get('Destination'))
next_passage = RealTimePassage(dt, direction)
next_passages.append(next_passage)
return next_passages
def _get_next_passage_for_route_point(self,
route_point,
count=None,
from_dt=None,
current_dt=None,
duration=None):
url = self._make_url(route_point, count, from_dt)
if not url:
return None
logging.getLogger(__name__).debug(
'Synthese RT service , call url : {}'.format(url))
r = self._call_synthese(url)
if not r:
return None
if r.status_code != 200:
# TODO better error handling, the response might be in 200 but in error
logging.getLogger(__name__).error(
'Synthese RT service unavailable, impossible to query : {}'.
format(r.url))
raise RealtimeProxyError('non 200 response')
return None
logging.getLogger(__name__).debug("synthese response: {}".format(
r.text))
passages = self._get_synthese_passages(r.content)
return self._find_route_point_passages(route_point, passages)
def _call(self, url):
self.log.debug('sirilite RT service, call url: {}'.format(url))
try:
return self.breaker.call(requests.get, url, timeout=self.timeout)
except pybreaker.CircuitBreakerError as e:
self.log.error(
'sirilite RT service dead, using base schedule (error: {}'.
format(e))
raise RealtimeProxyError('circuit breaker open')
except requests.Timeout as t:
self.log.error(
'sitelite RT service timeout, using base schedule (error: {}'.
format(t))
raise RealtimeProxyError('timeout')
except Exception as e:
self.log.exception('sirilite RT error, using base schedule')
raise RealtimeProxyError(str(e))
def _get_next_passage_for_route_point(self, route_point, count, from_dt, current_dt, duration=None):
stop = route_point.fetch_stop_id(self.object_id_tag)
request = self._make_request(monitoring_ref=stop, dt=from_dt, count=count)
if not request:
return None
siri_response = self._call_siri(request)
if not siri_response or siri_response.status_code != 200:
raise RealtimeProxyError('invalid response')
logging.getLogger(__name__).debug('siri for {}: {}'.format(stop, siri_response.text))
return self._get_passages(siri_response.content, route_point)
def _get_next_passage_for_route_point(
self, route_point, count=None, from_dt=None, current_dt=None, duration=None
):
params = self._make_params(route_point)
if not params:
return None
r = self._call(params)
if r.status_code != requests.codes.ok:
logging.getLogger(__name__).error(
'sytralrt service unavailable, impossible to query : {}'.format(r.url),
extra={'rt_system_id': unicode(self.rt_system_id)},
)
raise RealtimeProxyError('non 200 response')
return self._get_passages(route_point, r.json())
def _get_next_passage_for_route_point(self,
route_point,
count=None,
from_dt=None,
current_dt=None):
url = self._make_url(route_point)
if not url:
return None
r = self._call(url)
if r.status_code != 200:
self.log.error(
'sirilite RT service unavailable, impossible to query : {}'.
format(r.url))
raise RealtimeProxyError('non 200 response')
return self._get_passages(route_point, r.json())
def _get_next_passage_for_route_point(self,
route_point,
count=None,
from_dt=None,
current_dt=None,
duration=None):
url = self._make_url(route_point)
if not url:
return None
r = self._call_cleverage(url)
if not r:
return None
if r.status_code != 200:
# TODO better error handling, the response might be in 200 but in error
logging.getLogger(__name__).error(
'Cleverage RT service unavailable, impossible to query : {}'.
format(r.url))
raise RealtimeProxyError('non 200 response')
return self._get_passages(route_point, r.json())
def _get_passages(self, response, current_dt, line_uri=None):
status_code = response.status_code
timeo_resp = response.json()
logging.getLogger(__name__).debug(
'timeo response: {}'.format(timeo_resp),
extra={'rt_system_id': six.text_type(self.rt_system_id)})
# Handling http error
if status_code != 200:
logging.getLogger(__name__).error(
'Timeo RT service unavailable, impossible to query : {}'.
format(response.url),
extra={
'rt_system_id': six.text_type(self.rt_system_id),
'status_code': status_code
},
)
raise RealtimeProxyError('non 200 response')
# internal timeo error handling
message_responses = timeo_resp.get('MessageResponse')
for message_response in message_responses:
if ('ResponseCode' in message_response
and int(message_response['ResponseCode']) >=
self.INTERNAL_TIMEO_ERROR_CODE_LIMIT):
resp_code = message_response['ResponseCode']
if 'ResponseComment' in message_response:
resp_comment = message_response['ResponseComment']
else:
resp_comment = ''
self.record_internal_failure(
'Timeo RT internal service error',
'ResponseCode: {} - ResponseComment: {}'.format(
resp_code, resp_comment),
)
timeo_internal_error_message = 'Timeo RT internal service error, ResponseCode: {} - ResponseComment: {}'.format(
resp_code, resp_comment)
logging.getLogger(__name__).error(timeo_internal_error_message)
raise RealtimeProxyError(timeo_internal_error_message)
st_responses = timeo_resp.get('StopTimesResponse')
# by construction there should be only one StopTimesResponse
if not st_responses or len(st_responses) != 1:
logging.getLogger(__name__).warning(
'invalid timeo response: {}'.format(timeo_resp),
extra={'rt_system_id': six.text_type(self.rt_system_id)},
)
raise RealtimeProxyError('invalid response')
next_st = st_responses[0]['NextStopTimesMessage']
next_passages = []
for next_expected_st in next_st.get('NextExpectedStopTime', []):
# for the moment we handle only the NextStop and the direction
dt = self._get_dt(next_expected_st['NextStop'], current_dt)
direction = self._get_direction_name(
line_uri=line_uri,
object_code=next_expected_st.get('Terminus'),
default_value=next_expected_st.get('Destination'),
)
next_passage = RealTimePassage(dt, direction)
next_passages.append(next_passage)
return next_passages