def post(self):
raw_xml = get_ire(flask.globals.request)
# create a raw ire obj, save the raw_xml into the db
rt_update = _make_rt_update(raw_xml)
try:
# assuming UTF-8 encoding for all ire input
rt_update.raw_data = rt_update.raw_data.encode('utf-8')
# raw_xml is interpreted
trip_updates = KirinModelBuilder(self.navitia_wrapper, self.contributor).build(rt_update)
except KirinException as e:
rt_update.status = 'KO'
rt_update.error = e.data['error']
model.db.session.add(rt_update)
model.db.session.commit()
raise
except Exception as e:
rt_update.status = 'KO'
rt_update.error = e.message
model.db.session.add(rt_update)
model.db.session.commit()
raise
core.handle(rt_update, trip_updates, current_app.config['CONTRIBUTOR'])
return 'OK', 200
def post(self):
raw_xml = get_ire(flask.globals.request)
# create a raw ire obj, save the raw_xml into the db
rt_update = _make_rt_update(raw_xml)
# assuming UTF-8 encoding for all ire input
rt_update.raw_data = rt_update.raw_data.encode('utf-8')
# raw_xml is interpreted
trip_updates = KirinModelBuilder(make_navitia_wrapper()).build(rt_update)
core.handle(rt_update, trip_updates)
return 'OK', 200
def post(self):
raw_xml = get_ire(flask.globals.request)
# create a raw ire obj, save the raw_xml into the db
rt_update = _make_rt_update(raw_xml)
# assuming UTF-8 encoding for all ire input
rt_update.raw_data = rt_update.raw_data.encode('utf-8')
# raw_xml is interpreted
trip_updates = KirinModelBuilder(
make_navitia_wrapper(),
current_app.config['CONTRIBUTOR']).build(rt_update)
core.handle(rt_update, trip_updates, current_app.config['CONTRIBUTOR'])
return 'OK', 200
def handle(proto, navitia_wrapper, contributor):
data = str(proto) # temp, for the moment, we save the protobuf as text
rt_update = make_rt_update(data, 'gtfs-rt', contributor=contributor)
start_datetime = datetime.datetime.utcnow()
try:
trip_updates = KirinModelBuilder(navitia_wrapper,
contributor).build(rt_update,
data=proto)
record_call('OK', contributor=contributor)
except KirinException as e:
rt_update.status = 'KO'
rt_update.error = e.data['error']
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=contributor)
raise
except Exception as e:
rt_update.status = 'KO'
rt_update.error = e.message
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=contributor)
raise
real_time_update, log_dict = core.handle(rt_update, trip_updates,
contributor)
duration = (datetime.datetime.utcnow() - start_datetime).total_seconds()
log_dict.update({
'duration':
duration,
'input_timestamp':
datetime.datetime.utcfromtimestamp(proto.header.timestamp)
})
record_call('Simple feed publication', **log_dict)
logging.getLogger(__name__).info('Simple feed publication', extra=log_dict)
def handle(proto, navitia_wrapper, contributor):
data = str(proto) # temp, for the moment, we save the protobuf as text
rt_update = make_rt_update(data, 'gtfs-rt')
try:
trip_updates = KirinModelBuilder(navitia_wrapper, contributor).build(rt_update, data=proto)
except KirinException as e:
rt_update.status = 'KO'
rt_update.error = e.data['error']
model.db.session.add(rt_update)
model.db.session.commit()
raise
except Exception as e:
rt_update.status = 'KO'
rt_update.error = e.message
model.db.session.add(rt_update)
model.db.session.commit()
raise
core.handle(rt_update, trip_updates, contributor)
def handle(proto, navitia_wrapper, contributor):
data = str(proto) # temp, for the moment, we save the protobuf as text
rt_update = make_rt_update(data, 'gtfs-rt', contributor=contributor)
start_datetime = datetime.datetime.utcnow()
try:
trip_updates = KirinModelBuilder(navitia_wrapper,
contributor).build(rt_update,
data=proto)
record_call('OK', contributor=contributor)
except KirinException as e:
rt_update.status = 'KO'
rt_update.error = e.data['error']
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=contributor)
raise
except Exception as e:
rt_update.status = 'KO'
rt_update.error = e.message
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=contributor)
raise
real_time_update, log_dict = core.handle(rt_update, trip_updates,
contributor)
# After merging trip_updates information of gtfs-rt, navitia and kirin database, if there is no new information
# destinated to navitia, update real_time_update with status = 'KO' and a proper error message.
if not real_time_update.trip_updates and real_time_update.status == 'OK':
real_time_update.status = 'KO'
real_time_update.error = 'No new information destinated to navitia for this gtfs-rt ' \
'with timestamp: {}'.format(proto.header.timestamp)
logging.getLogger(__name__).error(
'No new information destinated to navitia for this gtfs-rt '
'with timestamp: {}'.format(proto.header.timestamp))
model.db.session.add(rt_update)
model.db.session.commit()
duration = (datetime.datetime.utcnow() - start_datetime).total_seconds()
log_dict.update({
'duration':
duration,
'input_timestamp':
datetime.datetime.utcfromtimestamp(proto.header.timestamp)
})
record_call('Simple feed publication', **log_dict)
logging.getLogger(__name__).info('Simple feed publication', extra=log_dict)
def process_post(self, input_raw, contributor_type, is_new_complete=False):
# create a raw rt_update obj, save the raw_input into the db
rt_update = make_rt_update(input_raw,
contributor_type,
contributor=self.contributor)
start_datetime = datetime.utcnow()
try:
# assuming UTF-8 encoding for all input
rt_update.raw_data = rt_update.raw_data.encode('utf-8')
# raw_input is interpreted
trip_updates = self.builder(self.navitia_wrapper,
self.contributor).build(rt_update)
record_call('OK', contributor=self.contributor)
except KirinException as e:
rt_update.status = 'KO'
rt_update.error = e.data['error']
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=self.contributor)
raise
except Exception as e:
rt_update.status = 'KO'
rt_update.error = e.message
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=self.contributor)
raise
_, log_dict = core.handle(rt_update,
trip_updates,
self.contributor,
is_new_complete=is_new_complete)
duration = (datetime.utcnow() - start_datetime).total_seconds()
log_dict.update({'duration': duration})
record_call('Simple feed publication', **log_dict)
logging.getLogger(__name__).info('Simple feed publication',
extra=log_dict)
return 'OK', 200
def post(self):
raw_xml = get_ire(flask.globals.request)
# create a raw ire obj, save the raw_xml into the db
rt_update = make_rt_update(raw_xml,
'ire',
contributor=self.contributor)
start_datetime = datetime.utcnow()
try:
# assuming UTF-8 encoding for all ire input
rt_update.raw_data = rt_update.raw_data.encode('utf-8')
# raw_xml is interpreted
trip_updates = KirinModelBuilder(self.navitia_wrapper,
self.contributor).build(rt_update)
record_call('OK', contributor=self.contributor)
except KirinException as e:
rt_update.status = 'KO'
rt_update.error = e.data['error']
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=self.contributor)
raise
except Exception as e:
rt_update.status = 'KO'
rt_update.error = e.message
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=self.contributor)
raise
_, log_dict = core.handle(rt_update, trip_updates,
current_app.config['CONTRIBUTOR'])
duration = (datetime.utcnow() - start_datetime).total_seconds()
log_dict.update({'duration': duration})
record_call('Simple feed publication', **log_dict)
logging.getLogger(__name__).info('Simple feed publication',
extra=log_dict)
return 'OK', 200
def test_get_action_on_trip_add(mock_navitia_fixture):
"""
Test the function _get_action_on_trip with different type of flux cots
returns:
1. Fist trip add(AJOUTEE)-> FIRST_TIME_ADDED
2. Add followed by update (PERTURBEE) -> PREVIOUSLY_ADDED
3. Delete followed by add -> FIRST_TIME_ADDED
"""
with app.app_context():
# Test for the first add: should be FIRST_TIME_ADDED
input_trip_add = get_fixture_data('cots_train_151515_added_trip.json')
json_data = json.loads(input_trip_add)
dict_version = model_maker.get_value(json_data, 'nouvelleVersion')
train_numbers = model_maker.get_value(dict_version, 'numeroCourse')
pdps = model_maker._retrieve_interesting_pdp(
model_maker.get_value(dict_version, 'listePointDeParcours'))
action_on_trip = model_maker._get_action_on_trip(
train_numbers, dict_version, pdps)
assert action_on_trip == ActionOnTrip.FIRST_TIME_ADDED.name
# Test for add followed by update should be PREVIOUSLY_ADDED
rt_update = model.RealTimeUpdate(input_trip_add,
connector='cots',
contributor='realtime.cots')
trip_updates = KirinModelBuilder(dumb_nav_wrapper()).build(rt_update)
_, log_dict = handle(rt_update,
trip_updates,
'realtime.cots',
is_new_complete=True)
input_update_added_trip = get_fixture_data(
'cots_train_151515_added_trip_with_delay.json')
json_data = json.loads(input_update_added_trip)
dict_version = model_maker.get_value(json_data, 'nouvelleVersion')
train_numbers = model_maker.get_value(dict_version, 'numeroCourse')
pdps = model_maker._retrieve_interesting_pdp(
model_maker.get_value(dict_version, 'listePointDeParcours'))
action_on_trip = model_maker._get_action_on_trip(
train_numbers, dict_version, pdps)
assert action_on_trip == ActionOnTrip.PREVIOUSLY_ADDED.name
# Clean database for further test
tables = [str(table) for table in db.metadata.sorted_tables]
db.session.execute('TRUNCATE {} CASCADE;'.format(', '.join(tables)))
db.session.commit()
# Delete the recently added trip followed by add: should be FIRST_TIME_ADDED
rt_update = model.RealTimeUpdate(input_trip_add,
connector='cots',
contributor='realtime.cots')
trip_updates = KirinModelBuilder(dumb_nav_wrapper()).build(rt_update)
_, log_dict = handle(rt_update,
trip_updates,
'realtime.cots',
is_new_complete=True)
input_trip_delete = get_fixture_data(
'cots_train_151515_deleted_trip_with_delay_and_stop_time_added.json'
)
rt_update = model.RealTimeUpdate(input_trip_delete,
connector='cots',
contributor='realtime.cots')
trip_updates = KirinModelBuilder(dumb_nav_wrapper()).build(rt_update)
_, log_dict = handle(rt_update,
trip_updates,
'realtime.cots',
is_new_complete=True)
input_added_trip = get_fixture_data(
'cots_train_151515_added_trip.json')
json_data = json.loads(input_added_trip)
dict_version = model_maker.get_value(json_data, 'nouvelleVersion')
train_numbers = model_maker.get_value(dict_version, 'numeroCourse')
pdps = model_maker._retrieve_interesting_pdp(
model_maker.get_value(dict_version, 'listePointDeParcours'))
action_on_trip = model_maker._get_action_on_trip(
train_numbers, dict_version, pdps)
assert action_on_trip == ActionOnTrip.FIRST_TIME_ADDED.name
def wrap_build(builder, input_raw):
"""
Function wrapping the processing of realtime information of an external feed
This manages errors/logger/newrelic
:param builder: the KirinModelBuilder to be called
:param input_raw: the feed to process
"""
contributor = builder.contributor
start_datetime = datetime.utcnow()
rt_update = None
log_dict = {"contributor": contributor.id}
status = "OK"
try:
# create a raw rt_update obj, save the raw_input into the db
rt_update, rtu_log_dict = builder.build_rt_update(input_raw)
log_dict.update(rtu_log_dict)
# raw_input is interpreted
trip_updates, tu_log_dict = builder.build_trip_updates(rt_update)
log_dict.update(tu_log_dict)
# finally confront to previously existing information (base_schedule, previous real-time)
_, handler_log_dict = core.handle(rt_update, trip_updates,
contributor.id,
builder.is_new_complete)
log_dict.update(handler_log_dict)
except Exception as e:
status = "failure"
allow_reprocess = True
if is_invalid_input_exception(e):
status = "warning" # Kirin did his job correctly if the input is invalid and rejected
allow_reprocess = False # reprocess is useless if input is invalid
if rt_update is not None:
error = e.data["error"] if (isinstance(e, KirinException)
and "error" in e.data) else e.message
set_rtu_status_ko(rt_update,
error,
is_reprocess_same_data_allowed=allow_reprocess)
model.db.session.add(rt_update)
model.db.session.commit()
else:
# rt_update is not built, make sure reprocess is allowed
allow_reprocess_same_data(contributor.id)
log_dict.update({"exc_summary": six.text_type(e), "reason": e})
record_custom_parameter(
"reason", e) # using __str__() here to have complete details
raise # filters later for APM (auto.)
finally:
log_dict.update(
{"duration": (datetime.utcnow() - start_datetime).total_seconds()})
record_call(status, **log_dict)
if status == "OK":
logging.getLogger(__name__).info(status, extra=log_dict)
elif status == "warning":
logging.getLogger(__name__).warning(status, extra=log_dict)
else:
logging.getLogger(__name__).error(status, extra=log_dict)
