def build_rt_update(self, input_raw): # create a raw gtfs-rt obj, save the raw protobuf into the db proto = gtfs_realtime_pb2.FeedMessage() log_dict = {} try: proto.ParseFromString(input_raw) except DecodeError: # We save the non-decodable flux gtfs-rt rt_update = manage_db_error( input_raw.encode( "string_escape", "ignore"), # protect for PostgreSQL "Text" type ConnectorType.gtfs_rt.value, contributor_id=self.contributor.id, error="invalid protobuf", is_reprocess_same_data_allowed=False, ) return rt_update, log_dict feed = six.binary_type( proto) # temp, for the moment, we save the protobuf as text rt_update = make_rt_update( feed, connector_type=self.contributor.connector_type, contributor_id=self.contributor.id) rt_update.proto = proto return rt_update, log_dict
def make_96231_20150728_0(): """gtfs rt feed corresponding to train_96231_delayed.xml""" message = gtfs_realtime_pb2.FeedMessage() message.header.incrementality = gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL entity = message.entity.add() entity.id = "96231_2015-07-28_0" trip_update = entity.trip_update trip = trip_update.trip trip.trip_id = "vehicle_journey:OCETrainTER-87212027-85000109-3:15554" trip.start_date = "20150728" trip.schedule_relationship = gtfs_realtime_pb2.TripDescriptor.SCHEDULED # Strasbourg stop_time = trip_update.stop_time_update.add() stop_time.stop_id = "stop_point:OCE:SP:TrainTER-87212027" stop_time.arrival.time = unix_time(2015, 07, 28, 17, 21, 0) stop_time.departure.time = unix_time(2015, 07, 28, 17, 21, 0) stop_time.stop_sequence = 1 # Sélestat stop_time = trip_update.stop_time_update.add() stop_time.stop_id = "stop_point:OCE:SP:TrainTER-87214056" stop_time.arrival.time = unix_time(2015, 07, 28, 17, 38, 0) stop_time.departure.time = unix_time(2015, 07, 28, 17, 40 + 15, 0) stop_time.stop_sequence = 2 # Colmar stop_time = trip_update.stop_time_update.add() stop_time.stop_id = "stop_point:OCE:SP:TrainTER-87182014" stop_time.arrival.time = unix_time(2015, 07, 28, 17, 51 + 15, 0) stop_time.departure.time = unix_time(2015, 07, 28, 17, 53 + 15, 0) stop_time.stop_sequence = 3 # Mulhouse stop_time = trip_update.stop_time_update.add() stop_time.stop_id = "stop_point:OCE:SP:TrainTER-87182063" stop_time.arrival.time = unix_time(2015, 07, 28, 18, 14 + 15, 0) stop_time.departure.time = unix_time(2015, 07, 28, 18, 16 + 15, 0) stop_time.stop_sequence = 4 # St-Louis stop_time = trip_update.stop_time_update.add() stop_time.stop_id = "stop_point:OCE:SP:TrainTER-87182139" stop_time.arrival.time = unix_time(2015, 07, 28, 18, 30 + 15, 0) stop_time.departure.time = unix_time(2015, 07, 28, 18, 31 + 15, 0) stop_time.stop_sequence = 6 # Basel-SBB stop_time = trip_update.stop_time_update.add() stop_time.stop_id = "stop_point:OCE:SP:TrainTER-85000109" stop_time.arrival.time = unix_time(2015, 07, 28, 18, 39 + 15, 0) stop_time.departure.time = unix_time(2015, 07, 28, 18, 39, 0)# ? stop_time.stop_sequence = 7 return message
def _convert_text_to_pbf(data): # late import for mypy from google.protobuf import text_format from kirin import gtfs_realtime_pb2 proto = gtfs_realtime_pb2.FeedMessage() text_format.Parse(data, proto) return proto.SerializeToString()
def _convert_json_to_pbf(data): # late import for mypy import ujson from google.protobuf import json_format from kirin import gtfs_realtime_pb2 proto = gtfs_realtime_pb2.FeedMessage() json_data = ujson.loads(data) json_format.ParseDict(json_data, proto) return proto.SerializeToString()
def convert_to_gtfsrt( trip_updates, incrementality=gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL): feed = gtfs_realtime_pb2.FeedMessage() feed.header.incrementality = incrementality feed.header.gtfs_realtime_version = "1" feed.header.timestamp = to_posix_time(datetime.datetime.utcnow()) for trip_update in trip_updates: fill_entity(feed.entity.add(), trip_update) return feed
def post(self): raw_proto = _get_gtfs_rt(flask.globals.request) from kirin import gtfs_realtime_pb2 # create a raw gtfs-rt obj, save the raw protobuf into the db proto = gtfs_realtime_pb2.FeedMessage() try: proto.ParseFromString(raw_proto) except DecodeError: raise InvalidArguments('invalid protobuf') proto.ParseFromString(raw_proto) model_maker.handle(proto, self.navitia_wrapper, self.contributor) return 'OK', 200
def gtfs_poller(self, config): func_name = 'gtfs_poller' logger = logging.LoggerAdapter(logging.getLogger(__name__), extra={'contributor': config['contributor']}) logger.debug('polling of %s', config['feed_url']) contributor = config['contributor'] lock_name = make_kirin_lock_name(func_name, contributor) with get_lock(logger, lock_name, app.config['REDIS_LOCK_TIMEOUT_POLLER']) as locked: if not locked: new_relic.ignore_transaction() return # We do a HEAD request at the very beginning of polling and we compare it with the previous one to check if # the gtfs-rt is changed. # If the HEAD request or Redis get/set fail, we just ignore this part and do the polling anyway if not _is_newer(config): new_relic.ignore_transaction() manage_db_no_new(connector='gtfs-rt', contributor=contributor) return try: response = requests.get(config['feed_url'], timeout=config.get('timeout', 1)) response.raise_for_status() except Exception as e: manage_db_error(data='', connector='gtfs-rt', contributor=contributor, status='KO', error='Http Error') logger.debug(str(e)) return nav = navitia_wrapper.Navitia(url=config['navitia_url'], token=config['token'], timeout=5, cache=redis, query_timeout=app.config.get('NAVITIA_QUERY_CACHE_TIMEOUT', 600), pubdate_timeout=app.config.get('NAVITIA_PUBDATE_CACHE_TIMEOUT', 600))\ .instance(config['coverage']) proto = gtfs_realtime_pb2.FeedMessage() try: proto.ParseFromString(response.content) except DecodeError: manage_db_error(proto, 'gtfs-rt', contributor=contributor, status='KO', error='Decode Error') logger.debug('invalid protobuf') else: model_maker.handle(proto, nav, contributor) logger.info('%s for %s is finished', func_name, contributor)
def gtfs_poller(self, config): logger = logging.LoggerAdapter( logging.getLogger(__name__), extra={'contributor': config['contributor']}) logger.debug('polling of %s', config['feed_url']) response = requests.get(config['feed_url'], timeout=config.get('timeout', 1)) response.raise_for_status() nav = navitia_wrapper.Navitia(url=config['navitia_url'], token=config['token'])\ .instance(config['coverage']) nav.timeout = 5 proto = gtfs_realtime_pb2.FeedMessage() proto.ParseFromString(response.content) model_maker.handle(proto, nav, config['contributor']) logger.debug('gtfsrt polling finished')
def post(self): raw_proto = _get_gtfs_rt(flask.globals.request) from kirin import gtfs_realtime_pb2 # create a raw gtfs-rt obj, save the raw protobuf into the db proto = gtfs_realtime_pb2.FeedMessage() try: proto.ParseFromString(raw_proto) except DecodeError: #We save the non-decodable flux gtfs-rt manage_db_error(proto, 'gtfs-rt', contributor=self.contributor, status='KO', error='Decode Error') raise InvalidArguments('invalid protobuf') else: model_maker.handle(proto, self.navitia_wrapper, self.contributor) return 'OK', 200
def build_rt_update(self, input_raw): # create a raw gtfs-rt obj, save the raw protobuf into the db (as text) proto = gtfs_realtime_pb2.FeedMessage() log_dict = {} try: proto.ParseFromString(input_raw) except DecodeError: # We save the non-decodable gtfs-rt feed rt_update = manage_db_error( str(input_raw, encoding="utf-8", errors="replace"), # protect for PostgreSQL "Text" type contributor_id=self.contributor.id, error="invalid protobuf", is_reprocess_same_data_allowed=False, ) return rt_update, log_dict feed = str(proto) # temp, for the moment, we save the protobuf as text rt_update = make_rt_update(feed, contributor_id=self.contributor.id) rt_update.proto = proto return rt_update, log_dict
def basic_gtfs_rt_data(): feed = gtfs_realtime_pb2.FeedMessage() feed.header.gtfs_realtime_version = "1.0" feed.header.incrementality = gtfs_realtime_pb2.FeedHeader.FULL_DATASET feed.header.timestamp = to_posix_time(datetime.datetime(year=2012, month=6, day=15, hour=15)) entity = feed.entity.add() entity.id = 'bob' trip_update = entity.trip_update trip_update.trip.trip_id = "Code-R-vj1" stu = trip_update.stop_time_update.add() stu.arrival.delay = 60 stu.stop_sequence = 2 stu.stop_id = "Code-StopR2" stu = trip_update.stop_time_update.add() stu.arrival.delay = 180 stu.stop_sequence = 4 stu.stop_id = "Code-StopR4" return feed