def _service_caller(self, method, url, headers, data=None):
try:
kwargs = {'timeout': self.timeout, 'headers': headers}
if data:
kwargs.update({"data": data})
response = self.breaker.call(method, url, **kwargs)
if not response or response.status_code != 200:
logging.getLogger(__name__).error(
'COTS cause message sub-service, Invalid response, '
'status_code: {}'.format(response.status_code))
if response.status_code == 401:
raise UnauthorizedOnSubService(
'Unauthorized on COTS message sub-service {} {}'.
format(method, url))
raise ObjectNotFound('non 200 response on COTS cause message '
'sub-service {} {}'.format(method, url))
return response
except pybreaker.CircuitBreakerError as e:
logging.getLogger(__name__).error(
'COTS cause message sub-service dead (error: {})'.format(e))
raise SubServiceError(
'COTS cause message sub-service circuit breaker open')
except requests.Timeout as t:
logging.getLogger(__name__).error(
'COTS cause message sub-service timeout (error: {})'.format(t))
raise SubServiceError('COTS cause message sub-service timeout')
except (UnauthorizedOnSubService, ObjectNotFound):
raise # Do not change exceptions that were just raised
except Exception as e:
logging.getLogger(__name__).exception(
'COTS cause message sub-service handling '
'error : {}'.format(str(e)))
raise SubServiceError(str(e))
def _service_caller(self, method, url, headers, data=None):
try:
kwargs = {"timeout": self.timeout, "headers": headers}
if data:
kwargs.update({"data": data})
response = method(url, **kwargs)
if not response or response.status_code != 200:
logging.getLogger(__name__).error(
"COTS cause message sub-service, Invalid response, "
"status_code: {}".format(response.status_code))
if response.status_code == 401:
raise UnauthorizedOnSubService(
"Unauthorized on COTS message sub-service {} {}".
format(method, url))
raise ObjectNotFound(
"non 200 response on COTS cause message sub-service {} {}".
format(method, url))
return response
except requests.Timeout as t:
logging.getLogger(__name__).error(
"COTS cause message sub-service timeout (error: {})".format(t))
raise SubServiceError("COTS cause message sub-service timeout")
except (UnauthorizedOnSubService, ObjectNotFound):
raise # Do not change exceptions that were just raised
except Exception as e:
logging.getLogger(__name__).exception(
"COTS cause message sub-service handling error : {}".format(
six.text_type(e)))
raise SubServiceError(six.text_type(e))
def __init__(self, navitia_vj, since_dt, until_dt, vj_start_dt=None):
"""
Create a circulation (VJ on a given day) from:
* the navitia VJ (circulation times without a specific day)
* a datetime that's close but BEFORE the start of the circulation considered
As Navitia doesn't return the start-timestamp that matches the search period (only a time, no date),
Kirin needs to re-process it here:
This processes start-timestamp of the circulation to be the closest one after since_dt.
day: 01 02 03 04
hour: 00:00 00:00 00:00 00:00
navitia VJ starts everyday: | 02:00 | 02:00 | 02:00 | 02:00
search period [since, until]: | | [23:00 09:00] |
| | | ^ |
actual start-timestamp: | | | 03T02:00 |
:param navitia_vj: json dict of navitia's response when looking for a VJ.
:param since_dt: naive UTC datetime BEFORE start of considered circulation,
typically the "since" parameter of the search in navitia.
:param until_dt: naive UTC datetime AFTER start of considered circulation,
typically the "until" parameter of the search in navitia.
:param vj_start_dt: naive UTC datetime of the first stop_time of vj.
"""
if (
since_dt.tzinfo is not None
or until_dt.tzinfo is not None
or (vj_start_dt is not None and vj_start_dt.tzinfo is not None)
):
raise InternalException("Invalid datetime provided: must be naive (and UTC)")
self.id = gen_uuid()
if "trip" in navitia_vj and "id" in navitia_vj["trip"]:
self.navitia_trip_id = navitia_vj["trip"]["id"]
# For an added trip, we use vj_start_dt as in flux cots whereas for an existing one
# compute start_timestamp (in UTC) from first stop_time, to be the closest AFTER provided since_dt.
if not navitia_vj.get("stop_times", None) and vj_start_dt:
self.start_timestamp = vj_start_dt
else:
first_stop_time = navitia_vj.get("stop_times", [{}])[0]
start_time = first_stop_time["utc_arrival_time"] # converted in datetime.time() in python wrapper
if start_time is None:
start_time = first_stop_time[
"utc_departure_time"
] # converted in datetime.time() in python wrapper
self.start_timestamp = datetime.datetime.combine(since_dt.date(), start_time)
# if since = 20010102T2300 and start_time = 0200, actual start_timestamp = 20010103T0200.
# So adding one day to start_timestamp obtained (20010102T0200) if it's before since_dt.
if self.start_timestamp < since_dt:
self.start_timestamp += timedelta(days=1)
# simple consistency check (for now): the start timestamp must also be BEFORE until_dt
if until_dt < self.start_timestamp:
msg = "impossible to calculate the circulation date of vj: {} on period [{}, {}]".format(
navitia_vj.get("id"), since_dt, until_dt
)
raise ObjectNotFound(msg)
self.navitia_vj = navitia_vj # Not persisted
def _get_navitia_company_id(self, code):
"""
Get a navitia company for the given code
"""
company = self._request_navitia_company(code)
if not company:
company = self._request_navitia_company(DEFAULT_COMPANY_CODE)
if not company:
raise ObjectNotFound(
"no company found for key {}, nor for the default company {}"
.format(code, DEFAULT_COMPANY_CODE))
return company.get("id", None) if company else None
def _get_navitia_vj(self, piv_key, train_date, ads, is_trip_addition):
self.log.debug("searching for vj {} in navitia".format(piv_key))
# large filter on date mostly to ensure only base-schedule VJ are requested
filter = 'vehicle_journey.has_code("rt_piv", "{}")'.format(piv_key)
train_datetime = datetime.datetime.combine(train_date,
datetime.time(0, 0, 0))
since_dt = train_datetime - datetime.timedelta(days=1)
until_dt = train_datetime + datetime.timedelta(days=2)
navitia_vjs = self._request_navitia_vehicle_journeys(filter,
since_dt,
until_dt,
depth=2,
show_codes=True)
if not navitia_vjs:
# Last PIV information is always right, so if the VJ doesn't exist, it's an ADD (no matter feed content)
navitia_vjs = [_make_navitia_empty_vj(piv_key)]
vj = None
if len(navitia_vjs) != 1:
self.log.info(
"Can not match a unique train for key {}".format(piv_key))
record_internal_failure("no unique train",
contributor=self.contributor.id)
else:
navitia_vj = navitia_vjs[0]
try:
base_vs_rt_error_margin = datetime.timedelta(hours=1)
vj_base_start = _get_first_stop_base_datetime(
ads, "depart", skip_fully_added_stops=not is_trip_addition)
# not even a single stop is a base-schedule stop
if not vj_base_start:
raise InvalidArguments(
"Whole trip is specified as pre-existing, but no stop is specified as pre-existing"
)
vj = model.VehicleJourney(
navitia_vj,
vj_base_start - base_vs_rt_error_margin,
vj_base_start + base_vs_rt_error_margin,
vj_start_dt=vj_base_start,
)
except InvalidArguments as i:
raise i
except Exception as e:
self.log.exception(
"Error while creating kirin VJ of {}: {}".format(
navitia_vjs[0].get("id"), e))
record_internal_failure("Error while creating kirin VJ",
contributor=self.contributor.id)
if not vj:
raise ObjectNotFound("no train found for key {}".format(piv_key))
return vj
def _get_navitia_vjs(self, headsign_str, since_dt, until_dt):
"""
Search for navitia's vehicle journeys with given headsigns, in the period provided
"""
log = logging.getLogger(__name__)
vjs = {}
# to get the date of the vj we use the start/end of the vj + some tolerance
# since the SNCF data and navitia data might not be synchronized
extended_since_dt = since_dt - timedelta(hours=1)
extended_until_dt = until_dt + timedelta(hours=1)
# using a set to deduplicate
# one headsign_str (ex: "96320/1") can lead to multiple headsigns (ex: ["96320", "96321"])
# but most of the time (if not always) they refer to the same VJ
# (the VJ switches headsign along the way).
# So we do one VJ search for each headsign to ensure we get it, then deduplicate VJs
for train_number in headsigns(headsign_str):
log.debug('searching for vj {} on {} in navitia'.format(
train_number, since_dt))
navitia_vjs = self.navitia.vehicle_journeys(
q={
'headsign': train_number,
'since': to_navitia_str(extended_since_dt),
'until': to_navitia_str(extended_until_dt),
'depth':
'2', # we need this depth to get the stoptime's stop_area
'show_codes':
'true' # we need the stop_points CRCICH codes
})
if not navitia_vjs:
logging.getLogger(__name__).info(
'impossible to find train {t} on [{s}, {u}['.format(
t=train_number,
s=extended_since_dt,
u=extended_until_dt))
record_internal_failure('missing train',
contributor=self.contributor)
for nav_vj in navitia_vjs:
vj = model.VehicleJourney(nav_vj, since_dt.date())
vjs[nav_vj['id']] = vj
if not vjs:
raise ObjectNotFound(
'no train found for headsign(s) {}'.format(headsign_str))
return vjs.values()
def _get_vjs(self, xml_train):
log = logging.getLogger(__name__)
train_numbers = headsigns(get_value(xml_train, 'NumeroTrain'))
# to get the date of the vj we use the start/end of the vj + some tolerance
# since the ire data and navitia data might not be synchronized
vj_start = as_date(
get_value(xml_train, 'OrigineTheoriqueTrain/DateHeureDepart'))
since = vj_start - timedelta(hours=1)
vj_end = as_date(
get_value(xml_train, 'TerminusTheoriqueTrain/DateHeureTerminus'))
until = vj_end + timedelta(hours=1)
vjs = {}
for train_number in train_numbers:
log.debug('searching for vj {} on {} in navitia'.format(
train_number, vj_start))
navitia_vjs = self.navitia.vehicle_journeys(
q={
'headsign': train_number,
'since': to_str(since),
'until': to_str(until),
'depth':
'2', # we need this depth to get the stoptime's stop_area
'show_codes':
'true' # we need the stop_points CRCICH codes
})
if not navitia_vjs:
logging.getLogger(__name__).info(
'impossible to find train {t} on [{s}, {u}['.format(
t=train_number, s=since, u=until))
record_internal_failure('missing train',
contributor=self.contributor)
for nav_vj in navitia_vjs:
vj = model.VehicleJourney(nav_vj, vj_start.date())
vjs[nav_vj['id']] = vj
if not vjs:
raise ObjectNotFound(
'no train found for headsigns {}'.format(train_numbers))
return vjs.values()
def _get_navitia_vjs(self,
headsign_str,
since_dt,
until_dt,
action_on_trip=ActionOnTrip.NOT_ADDED.name):
"""
Search for navitia's vehicle journeys with given headsigns, in the period provided
:param headsign_str: the headsigns to search for (can be multiple expressed in one string, like "2512/3")
:param since_dt: naive UTC datetime that starts the search period.
Typically the supposed datetime of first base-schedule stop_time.
:param until_dt: naive UTC datetime that ends the search period.
Typically the supposed datetime of last base-schedule stop_time.
:param action_on_trip: action to be performed on trip. This param is used to do consistency check
"""
if (since_dt is None) or (until_dt is None):
return []
if since_dt.tzinfo is not None or until_dt.tzinfo is not None:
raise InternalException(
"Invalid datetime provided: must be naive (and UTC)")
vjs = {}
# to get the date of the vj we use the start/end of the vj + some tolerance
# since the SNCF data and navitia data might not be synchronized
extended_since_dt = since_dt - SNCF_SEARCH_MARGIN
extended_until_dt = until_dt + SNCF_SEARCH_MARGIN
# using a set to deduplicate
# one headsign_str (ex: "96320/1") can lead to multiple headsigns (ex: ["96320", "96321"])
# but most of the time (if not always) they refer to the same VJ
# (the VJ switches headsign along the way).
# So we do one VJ search for each headsign to ensure we get it, then deduplicate VJs
for train_number in headsigns(headsign_str):
self.log.debug(
"searching for vj {} during period [{} - {}] in navitia".
format(train_number, extended_since_dt, extended_until_dt))
navitia_vjs = self.navitia.vehicle_journeys(
q={
"headsign": train_number,
"since": to_navitia_utc_str(extended_since_dt),
"until": to_navitia_utc_str(extended_until_dt),
"data_freshness": "base_schedule",
"depth":
"2", # we need this depth to get the stoptime's stop_area
"show_codes":
"true", # we need the stop_points CRCICH codes
})
# Consistency check on action applied to trip
if action_on_trip == ActionOnTrip.NOT_ADDED.name:
if not navitia_vjs:
self.log.info(
"impossible to find train {t} on [{s}, {u}[".format(
t=train_number,
s=extended_since_dt,
u=extended_until_dt))
record_internal_failure("missing train",
contributor=self.contributor.id)
else:
if action_on_trip == ActionOnTrip.FIRST_TIME_ADDED.name and navitia_vjs:
raise InvalidArguments(
"Invalid action, trip {} already present in navitia".
format(train_number))
navitia_vjs = [make_navitia_empty_vj(train_number)]
for nav_vj in navitia_vjs:
try:
vj = model.VehicleJourney(nav_vj,
extended_since_dt,
extended_until_dt,
vj_start_dt=since_dt)
vjs[nav_vj["id"]] = vj
except Exception as e:
self.log.exception(
"Error while creating kirin VJ of {}: {}".format(
nav_vj.get("id"), e))
record_internal_failure("Error while creating kirin VJ",
contributor=self.contributor.id)
if not vjs:
raise ObjectNotFound(
"no train found for headsign(s) {}".format(headsign_str))
return vjs.values()
def _get_navitia_vjs(self,
headsign_str,
utc_since_dt,
utc_until_dt,
action_on_trip=ActionOnTrip.NOT_ADDED.name):
"""
Search for navitia's vehicle journeys with given headsigns, in the period provided
:param utc_since_dt: UTC datetime that starts the search period.
Typically the supposed datetime of first base-schedule stop_time.
:param utc_until_dt: UTC datetime that ends the search period.
Typically the supposed datetime of last base-schedule stop_time.
"""
log = logging.getLogger(__name__)
if (utc_since_dt is None) or (utc_until_dt is None):
return []
vjs = {}
# to get the date of the vj we use the start/end of the vj + some tolerance
# since the SNCF data and navitia data might not be synchronized
extended_since_dt = utc_since_dt - SNCF_SEARCH_MARGIN
extended_until_dt = utc_until_dt + SNCF_SEARCH_MARGIN
# using a set to deduplicate
# one headsign_str (ex: "96320/1") can lead to multiple headsigns (ex: ["96320", "96321"])
# but most of the time (if not always) they refer to the same VJ
# (the VJ switches headsign along the way).
# So we do one VJ search for each headsign to ensure we get it, then deduplicate VJs
for train_number in headsigns(headsign_str):
log.debug('searching for vj {} during period [{} - {}] in navitia'.
format(train_number, extended_since_dt,
extended_until_dt))
navitia_vjs = self.navitia.vehicle_journeys(
q={
'headsign': train_number,
'since': to_navitia_str(extended_since_dt),
'until': to_navitia_str(extended_until_dt),
'depth':
'2', # we need this depth to get the stoptime's stop_area
'show_codes':
'true' # we need the stop_points CRCICH codes
})
if action_on_trip == ActionOnTrip.NOT_ADDED.name:
if not navitia_vjs:
logging.getLogger(__name__).info(
'impossible to find train {t} on [{s}, {u}['.format(
t=train_number,
s=extended_since_dt,
u=extended_until_dt))
record_internal_failure('missing train',
contributor=self.contributor)
else:
if action_on_trip == ActionOnTrip.FIRST_TIME_ADDED.name and navitia_vjs:
raise InvalidArguments(
'Invalid action, trip {} already present in navitia'.
format(train_number))
navitia_vjs = [make_navitia_empty_vj(train_number)]
for nav_vj in navitia_vjs:
try:
vj = model.VehicleJourney(nav_vj,
extended_since_dt,
extended_until_dt,
vj_start_dt=utc_since_dt)
vjs[nav_vj['id']] = vj
except Exception as e:
logging.getLogger(__name__).exception(
'Error while creating kirin VJ of {}: {}'.format(
nav_vj.get('id'), e))
record_internal_failure('Error while creating kirin VJ',
contributor=self.contributor)
if not vjs:
raise ObjectNotFound(
'no train found for headsign(s) {}'.format(headsign_str))
return vjs.values()