def build_rt_update(self, input_raw):
# create a raw gtfs-rt obj, save the raw protobuf into the db
proto = gtfs_realtime_pb2.FeedMessage()
log_dict = {}
try:
proto.ParseFromString(input_raw)
except DecodeError:
# We save the non-decodable flux gtfs-rt
rt_update = manage_db_error(
input_raw.encode(
"string_escape",
"ignore"), # protect for PostgreSQL "Text" type
ConnectorType.gtfs_rt.value,
contributor_id=self.contributor.id,
error="invalid protobuf",
is_reprocess_same_data_allowed=False,
)
return rt_update, log_dict
feed = six.binary_type(
proto) # temp, for the moment, we save the protobuf as text
rt_update = make_rt_update(
feed,
connector_type=self.contributor.connector_type,
contributor_id=self.contributor.id)
rt_update.proto = proto
return rt_update, log_dict
def make_96231_20150728_0():
"""gtfs rt feed corresponding to train_96231_delayed.xml"""
message = gtfs_realtime_pb2.FeedMessage()
message.header.incrementality = gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL
entity = message.entity.add()
entity.id = "96231_2015-07-28_0"
trip_update = entity.trip_update
trip = trip_update.trip
trip.trip_id = "vehicle_journey:OCETrainTER-87212027-85000109-3:15554"
trip.start_date = "20150728"
trip.schedule_relationship = gtfs_realtime_pb2.TripDescriptor.SCHEDULED
# Strasbourg
stop_time = trip_update.stop_time_update.add()
stop_time.stop_id = "stop_point:OCE:SP:TrainTER-87212027"
stop_time.arrival.time = unix_time(2015, 07, 28, 17, 21, 0)
stop_time.departure.time = unix_time(2015, 07, 28, 17, 21, 0)
stop_time.stop_sequence = 1
# Sélestat
stop_time = trip_update.stop_time_update.add()
stop_time.stop_id = "stop_point:OCE:SP:TrainTER-87214056"
stop_time.arrival.time = unix_time(2015, 07, 28, 17, 38, 0)
stop_time.departure.time = unix_time(2015, 07, 28, 17, 40 + 15, 0)
stop_time.stop_sequence = 2
# Colmar
stop_time = trip_update.stop_time_update.add()
stop_time.stop_id = "stop_point:OCE:SP:TrainTER-87182014"
stop_time.arrival.time = unix_time(2015, 07, 28, 17, 51 + 15, 0)
stop_time.departure.time = unix_time(2015, 07, 28, 17, 53 + 15, 0)
stop_time.stop_sequence = 3
# Mulhouse
stop_time = trip_update.stop_time_update.add()
stop_time.stop_id = "stop_point:OCE:SP:TrainTER-87182063"
stop_time.arrival.time = unix_time(2015, 07, 28, 18, 14 + 15, 0)
stop_time.departure.time = unix_time(2015, 07, 28, 18, 16 + 15, 0)
stop_time.stop_sequence = 4
# St-Louis
stop_time = trip_update.stop_time_update.add()
stop_time.stop_id = "stop_point:OCE:SP:TrainTER-87182139"
stop_time.arrival.time = unix_time(2015, 07, 28, 18, 30 + 15, 0)
stop_time.departure.time = unix_time(2015, 07, 28, 18, 31 + 15, 0)
stop_time.stop_sequence = 6
# Basel-SBB
stop_time = trip_update.stop_time_update.add()
stop_time.stop_id = "stop_point:OCE:SP:TrainTER-85000109"
stop_time.arrival.time = unix_time(2015, 07, 28, 18, 39 + 15, 0)
stop_time.departure.time = unix_time(2015, 07, 28, 18, 39, 0)# ?
stop_time.stop_sequence = 7
return message
def _convert_text_to_pbf(data):
# late import for mypy
from google.protobuf import text_format
from kirin import gtfs_realtime_pb2
proto = gtfs_realtime_pb2.FeedMessage()
text_format.Parse(data, proto)
return proto.SerializeToString()
def _convert_json_to_pbf(data):
# late import for mypy
import ujson
from google.protobuf import json_format
from kirin import gtfs_realtime_pb2
proto = gtfs_realtime_pb2.FeedMessage()
json_data = ujson.loads(data)
json_format.ParseDict(json_data, proto)
return proto.SerializeToString()
def convert_to_gtfsrt(
trip_updates,
incrementality=gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL):
feed = gtfs_realtime_pb2.FeedMessage()
feed.header.incrementality = incrementality
feed.header.gtfs_realtime_version = "1"
feed.header.timestamp = to_posix_time(datetime.datetime.utcnow())
for trip_update in trip_updates:
fill_entity(feed.entity.add(), trip_update)
return feed
def post(self):
raw_proto = _get_gtfs_rt(flask.globals.request)
from kirin import gtfs_realtime_pb2
# create a raw gtfs-rt obj, save the raw protobuf into the db
proto = gtfs_realtime_pb2.FeedMessage()
try:
proto.ParseFromString(raw_proto)
except DecodeError:
raise InvalidArguments('invalid protobuf')
proto.ParseFromString(raw_proto)
model_maker.handle(proto, self.navitia_wrapper, self.contributor)
return 'OK', 200
def gtfs_poller(self, config):
func_name = 'gtfs_poller'
logger = logging.LoggerAdapter(logging.getLogger(__name__), extra={'contributor': config['contributor']})
logger.debug('polling of %s', config['feed_url'])
contributor = config['contributor']
lock_name = make_kirin_lock_name(func_name, contributor)
with get_lock(logger, lock_name, app.config['REDIS_LOCK_TIMEOUT_POLLER']) as locked:
if not locked:
new_relic.ignore_transaction()
return
# We do a HEAD request at the very beginning of polling and we compare it with the previous one to check if
# the gtfs-rt is changed.
# If the HEAD request or Redis get/set fail, we just ignore this part and do the polling anyway
if not _is_newer(config):
new_relic.ignore_transaction()
manage_db_no_new(connector='gtfs-rt', contributor=contributor)
return
try:
response = requests.get(config['feed_url'], timeout=config.get('timeout', 1))
response.raise_for_status()
except Exception as e:
manage_db_error(data='', connector='gtfs-rt', contributor=contributor,
status='KO', error='Http Error')
logger.debug(str(e))
return
nav = navitia_wrapper.Navitia(url=config['navitia_url'],
token=config['token'],
timeout=5,
cache=redis,
query_timeout=app.config.get('NAVITIA_QUERY_CACHE_TIMEOUT', 600),
pubdate_timeout=app.config.get('NAVITIA_PUBDATE_CACHE_TIMEOUT', 600))\
.instance(config['coverage'])
proto = gtfs_realtime_pb2.FeedMessage()
try:
proto.ParseFromString(response.content)
except DecodeError:
manage_db_error(proto, 'gtfs-rt', contributor=contributor, status='KO', error='Decode Error')
logger.debug('invalid protobuf')
else:
model_maker.handle(proto, nav, contributor)
logger.info('%s for %s is finished', func_name, contributor)
def gtfs_poller(self, config):
logger = logging.LoggerAdapter(
logging.getLogger(__name__),
extra={'contributor': config['contributor']})
logger.debug('polling of %s', config['feed_url'])
response = requests.get(config['feed_url'],
timeout=config.get('timeout', 1))
response.raise_for_status()
nav = navitia_wrapper.Navitia(url=config['navitia_url'], token=config['token'])\
.instance(config['coverage'])
nav.timeout = 5
proto = gtfs_realtime_pb2.FeedMessage()
proto.ParseFromString(response.content)
model_maker.handle(proto, nav, config['contributor'])
logger.debug('gtfsrt polling finished')
def post(self):
raw_proto = _get_gtfs_rt(flask.globals.request)
from kirin import gtfs_realtime_pb2
# create a raw gtfs-rt obj, save the raw protobuf into the db
proto = gtfs_realtime_pb2.FeedMessage()
try:
proto.ParseFromString(raw_proto)
except DecodeError:
#We save the non-decodable flux gtfs-rt
manage_db_error(proto,
'gtfs-rt',
contributor=self.contributor,
status='KO',
error='Decode Error')
raise InvalidArguments('invalid protobuf')
else:
model_maker.handle(proto, self.navitia_wrapper, self.contributor)
return 'OK', 200
def build_rt_update(self, input_raw):
# create a raw gtfs-rt obj, save the raw protobuf into the db (as text)
proto = gtfs_realtime_pb2.FeedMessage()
log_dict = {}
try:
proto.ParseFromString(input_raw)
except DecodeError:
# We save the non-decodable gtfs-rt feed
rt_update = manage_db_error(
str(input_raw, encoding="utf-8", errors="replace"), # protect for PostgreSQL "Text" type
contributor_id=self.contributor.id,
error="invalid protobuf",
is_reprocess_same_data_allowed=False,
)
return rt_update, log_dict
feed = str(proto) # temp, for the moment, we save the protobuf as text
rt_update = make_rt_update(feed, contributor_id=self.contributor.id)
rt_update.proto = proto
return rt_update, log_dict
def basic_gtfs_rt_data():
feed = gtfs_realtime_pb2.FeedMessage()
feed.header.gtfs_realtime_version = "1.0"
feed.header.incrementality = gtfs_realtime_pb2.FeedHeader.FULL_DATASET
feed.header.timestamp = to_posix_time(datetime.datetime(year=2012, month=6, day=15, hour=15))
entity = feed.entity.add()
entity.id = 'bob'
trip_update = entity.trip_update
trip_update.trip.trip_id = "Code-R-vj1"
stu = trip_update.stop_time_update.add()
stu.arrival.delay = 60
stu.stop_sequence = 2
stu.stop_id = "Code-StopR2"
stu = trip_update.stop_time_update.add()
stu.arrival.delay = 180
stu.stop_sequence = 4
stu.stop_id = "Code-StopR4"
return feed