def post(self): raw_xml = get_ire(flask.globals.request) # create a raw ire obj, save the raw_xml into the db rt_update = _make_rt_update(raw_xml) try: # assuming UTF-8 encoding for all ire input rt_update.raw_data = rt_update.raw_data.encode('utf-8') # raw_xml is interpreted trip_updates = KirinModelBuilder(self.navitia_wrapper, self.contributor).build(rt_update) except KirinException as e: rt_update.status = 'KO' rt_update.error = e.data['error'] model.db.session.add(rt_update) model.db.session.commit() raise except Exception as e: rt_update.status = 'KO' rt_update.error = e.message model.db.session.add(rt_update) model.db.session.commit() raise core.handle(rt_update, trip_updates, current_app.config['CONTRIBUTOR']) return 'OK', 200
def post(self): raw_xml = get_ire(flask.globals.request) # create a raw ire obj, save the raw_xml into the db rt_update = _make_rt_update(raw_xml) # assuming UTF-8 encoding for all ire input rt_update.raw_data = rt_update.raw_data.encode('utf-8') # raw_xml is interpreted trip_updates = KirinModelBuilder(make_navitia_wrapper()).build(rt_update) core.handle(rt_update, trip_updates) return 'OK', 200
def post(self): raw_xml = get_ire(flask.globals.request) # create a raw ire obj, save the raw_xml into the db rt_update = _make_rt_update(raw_xml) # assuming UTF-8 encoding for all ire input rt_update.raw_data = rt_update.raw_data.encode('utf-8') # raw_xml is interpreted trip_updates = KirinModelBuilder( make_navitia_wrapper(), current_app.config['CONTRIBUTOR']).build(rt_update) core.handle(rt_update, trip_updates, current_app.config['CONTRIBUTOR']) return 'OK', 200
def handle(proto, navitia_wrapper, contributor): data = str(proto) # temp, for the moment, we save the protobuf as text rt_update = make_rt_update(data, 'gtfs-rt', contributor=contributor) start_datetime = datetime.datetime.utcnow() try: trip_updates = KirinModelBuilder(navitia_wrapper, contributor).build(rt_update, data=proto) record_call('OK', contributor=contributor) except KirinException as e: rt_update.status = 'KO' rt_update.error = e.data['error'] model.db.session.add(rt_update) model.db.session.commit() record_call('failure', reason=str(e), contributor=contributor) raise except Exception as e: rt_update.status = 'KO' rt_update.error = e.message model.db.session.add(rt_update) model.db.session.commit() record_call('failure', reason=str(e), contributor=contributor) raise real_time_update, log_dict = core.handle(rt_update, trip_updates, contributor) duration = (datetime.datetime.utcnow() - start_datetime).total_seconds() log_dict.update({ 'duration': duration, 'input_timestamp': datetime.datetime.utcfromtimestamp(proto.header.timestamp) }) record_call('Simple feed publication', **log_dict) logging.getLogger(__name__).info('Simple feed publication', extra=log_dict)
def handle(proto, navitia_wrapper, contributor): data = str(proto) # temp, for the moment, we save the protobuf as text rt_update = make_rt_update(data, 'gtfs-rt') try: trip_updates = KirinModelBuilder(navitia_wrapper, contributor).build(rt_update, data=proto) except KirinException as e: rt_update.status = 'KO' rt_update.error = e.data['error'] model.db.session.add(rt_update) model.db.session.commit() raise except Exception as e: rt_update.status = 'KO' rt_update.error = e.message model.db.session.add(rt_update) model.db.session.commit() raise core.handle(rt_update, trip_updates, contributor)
def handle(proto, navitia_wrapper, contributor): data = str(proto) # temp, for the moment, we save the protobuf as text rt_update = make_rt_update(data, 'gtfs-rt', contributor=contributor) start_datetime = datetime.datetime.utcnow() try: trip_updates = KirinModelBuilder(navitia_wrapper, contributor).build(rt_update, data=proto) record_call('OK', contributor=contributor) except KirinException as e: rt_update.status = 'KO' rt_update.error = e.data['error'] model.db.session.add(rt_update) model.db.session.commit() record_call('failure', reason=str(e), contributor=contributor) raise except Exception as e: rt_update.status = 'KO' rt_update.error = e.message model.db.session.add(rt_update) model.db.session.commit() record_call('failure', reason=str(e), contributor=contributor) raise real_time_update, log_dict = core.handle(rt_update, trip_updates, contributor) # After merging trip_updates information of gtfs-rt, navitia and kirin database, if there is no new information # destinated to navitia, update real_time_update with status = 'KO' and a proper error message. if not real_time_update.trip_updates and real_time_update.status == 'OK': real_time_update.status = 'KO' real_time_update.error = 'No new information destinated to navitia for this gtfs-rt ' \ 'with timestamp: {}'.format(proto.header.timestamp) logging.getLogger(__name__).error( 'No new information destinated to navitia for this gtfs-rt ' 'with timestamp: {}'.format(proto.header.timestamp)) model.db.session.add(rt_update) model.db.session.commit() duration = (datetime.datetime.utcnow() - start_datetime).total_seconds() log_dict.update({ 'duration': duration, 'input_timestamp': datetime.datetime.utcfromtimestamp(proto.header.timestamp) }) record_call('Simple feed publication', **log_dict) logging.getLogger(__name__).info('Simple feed publication', extra=log_dict)
def process_post(self, input_raw, contributor_type, is_new_complete=False): # create a raw rt_update obj, save the raw_input into the db rt_update = make_rt_update(input_raw, contributor_type, contributor=self.contributor) start_datetime = datetime.utcnow() try: # assuming UTF-8 encoding for all input rt_update.raw_data = rt_update.raw_data.encode('utf-8') # raw_input is interpreted trip_updates = self.builder(self.navitia_wrapper, self.contributor).build(rt_update) record_call('OK', contributor=self.contributor) except KirinException as e: rt_update.status = 'KO' rt_update.error = e.data['error'] model.db.session.add(rt_update) model.db.session.commit() record_call('failure', reason=str(e), contributor=self.contributor) raise except Exception as e: rt_update.status = 'KO' rt_update.error = e.message model.db.session.add(rt_update) model.db.session.commit() record_call('failure', reason=str(e), contributor=self.contributor) raise _, log_dict = core.handle(rt_update, trip_updates, self.contributor, is_new_complete=is_new_complete) duration = (datetime.utcnow() - start_datetime).total_seconds() log_dict.update({'duration': duration}) record_call('Simple feed publication', **log_dict) logging.getLogger(__name__).info('Simple feed publication', extra=log_dict) return 'OK', 200
def post(self): raw_xml = get_ire(flask.globals.request) # create a raw ire obj, save the raw_xml into the db rt_update = make_rt_update(raw_xml, 'ire', contributor=self.contributor) start_datetime = datetime.utcnow() try: # assuming UTF-8 encoding for all ire input rt_update.raw_data = rt_update.raw_data.encode('utf-8') # raw_xml is interpreted trip_updates = KirinModelBuilder(self.navitia_wrapper, self.contributor).build(rt_update) record_call('OK', contributor=self.contributor) except KirinException as e: rt_update.status = 'KO' rt_update.error = e.data['error'] model.db.session.add(rt_update) model.db.session.commit() record_call('failure', reason=str(e), contributor=self.contributor) raise except Exception as e: rt_update.status = 'KO' rt_update.error = e.message model.db.session.add(rt_update) model.db.session.commit() record_call('failure', reason=str(e), contributor=self.contributor) raise _, log_dict = core.handle(rt_update, trip_updates, current_app.config['CONTRIBUTOR']) duration = (datetime.utcnow() - start_datetime).total_seconds() log_dict.update({'duration': duration}) record_call('Simple feed publication', **log_dict) logging.getLogger(__name__).info('Simple feed publication', extra=log_dict) return 'OK', 200
def test_get_action_on_trip_add(mock_navitia_fixture): """ Test the function _get_action_on_trip with different type of flux cots returns: 1. Fist trip add(AJOUTEE)-> FIRST_TIME_ADDED 2. Add followed by update (PERTURBEE) -> PREVIOUSLY_ADDED 3. Delete followed by add -> FIRST_TIME_ADDED """ with app.app_context(): # Test for the first add: should be FIRST_TIME_ADDED input_trip_add = get_fixture_data('cots_train_151515_added_trip.json') json_data = json.loads(input_trip_add) dict_version = model_maker.get_value(json_data, 'nouvelleVersion') train_numbers = model_maker.get_value(dict_version, 'numeroCourse') pdps = model_maker._retrieve_interesting_pdp( model_maker.get_value(dict_version, 'listePointDeParcours')) action_on_trip = model_maker._get_action_on_trip( train_numbers, dict_version, pdps) assert action_on_trip == ActionOnTrip.FIRST_TIME_ADDED.name # Test for add followed by update should be PREVIOUSLY_ADDED rt_update = model.RealTimeUpdate(input_trip_add, connector='cots', contributor='realtime.cots') trip_updates = KirinModelBuilder(dumb_nav_wrapper()).build(rt_update) _, log_dict = handle(rt_update, trip_updates, 'realtime.cots', is_new_complete=True) input_update_added_trip = get_fixture_data( 'cots_train_151515_added_trip_with_delay.json') json_data = json.loads(input_update_added_trip) dict_version = model_maker.get_value(json_data, 'nouvelleVersion') train_numbers = model_maker.get_value(dict_version, 'numeroCourse') pdps = model_maker._retrieve_interesting_pdp( model_maker.get_value(dict_version, 'listePointDeParcours')) action_on_trip = model_maker._get_action_on_trip( train_numbers, dict_version, pdps) assert action_on_trip == ActionOnTrip.PREVIOUSLY_ADDED.name # Clean database for further test tables = [str(table) for table in db.metadata.sorted_tables] db.session.execute('TRUNCATE {} CASCADE;'.format(', '.join(tables))) db.session.commit() # Delete the recently added trip followed by add: should be FIRST_TIME_ADDED rt_update = model.RealTimeUpdate(input_trip_add, connector='cots', contributor='realtime.cots') trip_updates = KirinModelBuilder(dumb_nav_wrapper()).build(rt_update) _, log_dict = handle(rt_update, trip_updates, 'realtime.cots', is_new_complete=True) input_trip_delete = get_fixture_data( 'cots_train_151515_deleted_trip_with_delay_and_stop_time_added.json' ) rt_update = model.RealTimeUpdate(input_trip_delete, connector='cots', contributor='realtime.cots') trip_updates = KirinModelBuilder(dumb_nav_wrapper()).build(rt_update) _, log_dict = handle(rt_update, trip_updates, 'realtime.cots', is_new_complete=True) input_added_trip = get_fixture_data( 'cots_train_151515_added_trip.json') json_data = json.loads(input_added_trip) dict_version = model_maker.get_value(json_data, 'nouvelleVersion') train_numbers = model_maker.get_value(dict_version, 'numeroCourse') pdps = model_maker._retrieve_interesting_pdp( model_maker.get_value(dict_version, 'listePointDeParcours')) action_on_trip = model_maker._get_action_on_trip( train_numbers, dict_version, pdps) assert action_on_trip == ActionOnTrip.FIRST_TIME_ADDED.name
def wrap_build(builder, input_raw): """ Function wrapping the processing of realtime information of an external feed This manages errors/logger/newrelic :param builder: the KirinModelBuilder to be called :param input_raw: the feed to process """ contributor = builder.contributor start_datetime = datetime.utcnow() rt_update = None log_dict = {"contributor": contributor.id} status = "OK" try: # create a raw rt_update obj, save the raw_input into the db rt_update, rtu_log_dict = builder.build_rt_update(input_raw) log_dict.update(rtu_log_dict) # raw_input is interpreted trip_updates, tu_log_dict = builder.build_trip_updates(rt_update) log_dict.update(tu_log_dict) # finally confront to previously existing information (base_schedule, previous real-time) _, handler_log_dict = core.handle(rt_update, trip_updates, contributor.id, builder.is_new_complete) log_dict.update(handler_log_dict) except Exception as e: status = "failure" allow_reprocess = True if is_invalid_input_exception(e): status = "warning" # Kirin did his job correctly if the input is invalid and rejected allow_reprocess = False # reprocess is useless if input is invalid if rt_update is not None: error = e.data["error"] if (isinstance(e, KirinException) and "error" in e.data) else e.message set_rtu_status_ko(rt_update, error, is_reprocess_same_data_allowed=allow_reprocess) model.db.session.add(rt_update) model.db.session.commit() else: # rt_update is not built, make sure reprocess is allowed allow_reprocess_same_data(contributor.id) log_dict.update({"exc_summary": six.text_type(e), "reason": e}) record_custom_parameter( "reason", e) # using __str__() here to have complete details raise # filters later for APM (auto.) finally: log_dict.update( {"duration": (datetime.utcnow() - start_datetime).total_seconds()}) record_call(status, **log_dict) if status == "OK": logging.getLogger(__name__).info(status, extra=log_dict) elif status == "warning": logging.getLogger(__name__).warning(status, extra=log_dict) else: logging.getLogger(__name__).error(status, extra=log_dict)