def _on_request(self, message):
log = logging.getLogger(__name__)
try:
task = task_pb2.Task()
try:
# `body` is of unicode type, but we need str type for
# `ParseFromString()` to work. It seems to work.
# Maybe kombu estimate that, without any information,
# the body should be something as json, and thus a
# unicode string. On the c++ side, I didn't manage to
# find a way to give a content-type or something like
# that.
body = str(message.payload)
task.ParseFromString(body)
except DecodeError as e:
log.warn('invalid protobuf: {}'.format(str(e)))
return
log.info('Getting a full feed publication request', extra={'task': task})
if task.action != task_pb2.LOAD_REALTIME or not task.load_realtime:
return
start_datetime = datetime.utcnow()
begin_date = None
end_date = None
if hasattr(task.load_realtime, "begin_date"):
if task.load_realtime.begin_date:
begin_date = str_to_date(task.load_realtime.begin_date)
if hasattr(task.load_realtime, "end_date"):
if task.load_realtime.end_date:
end_date = str_to_date(task.load_realtime.end_date)
feed = convert_to_gtfsrt(TripUpdate.find_by_contributor_period(task.load_realtime.contributors,
begin_date,
end_date),
gtfs_realtime_pb2.FeedHeader.FULL_DATASET)
feed_str = feed.SerializeToString()
log.info('Starting of full feed publication {}, {}'.format(len(feed_str), task), extra={'size': len(feed_str), 'task': task})
# http://docs.celeryproject.org/projects/kombu/en/latest/userguide/producers.html#bypassing-routing-by-using-the-anon-exchange
self.producer.publish(feed_str,
routing_key=task.load_realtime.queue_name,
retry=True,
retry_policy={
'interval_start': 0, # First retry immediately,
'interval_step': 2, # then increase by 2s for every retry.
'interval_max': 10, # but don't exceed 10s between retries.
'max_retries': self.max_retries, # give up after 10 (by default) tries.
})
duration = (datetime.utcnow() - start_datetime).total_seconds()
log.info('End of full feed publication', extra={'duration': duration, 'task': task})
record_call('Full feed publication', size=len(feed_str), routing_key=task.load_realtime.queue_name,
duration=duration, trip_update_count=len(feed.entity),
contributor=task.load_realtime.contributors)
finally:
db.session.remove()
def handle(proto, navitia_wrapper, contributor):
data = str(proto) # temp, for the moment, we save the protobuf as text
rt_update = make_rt_update(data, 'gtfs-rt', contributor=contributor)
start_datetime = datetime.datetime.utcnow()
try:
trip_updates = KirinModelBuilder(navitia_wrapper,
contributor).build(rt_update,
data=proto)
record_call('OK', contributor=contributor)
except KirinException as e:
rt_update.status = 'KO'
rt_update.error = e.data['error']
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=contributor)
raise
except Exception as e:
rt_update.status = 'KO'
rt_update.error = e.message
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=contributor)
raise
real_time_update, log_dict = core.handle(rt_update, trip_updates,
contributor)
duration = (datetime.datetime.utcnow() - start_datetime).total_seconds()
log_dict.update({
'duration':
duration,
'input_timestamp':
datetime.datetime.utcfromtimestamp(proto.header.timestamp)
})
record_call('Simple feed publication', **log_dict)
logging.getLogger(__name__).info('Simple feed publication', extra=log_dict)
def handle(proto, navitia_wrapper, contributor):
data = str(proto) # temp, for the moment, we save the protobuf as text
rt_update = make_rt_update(data, 'gtfs-rt', contributor=contributor)
start_datetime = datetime.datetime.utcnow()
try:
trip_updates = KirinModelBuilder(navitia_wrapper,
contributor).build(rt_update,
data=proto)
record_call('OK', contributor=contributor)
except KirinException as e:
rt_update.status = 'KO'
rt_update.error = e.data['error']
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=contributor)
raise
except Exception as e:
rt_update.status = 'KO'
rt_update.error = e.message
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=contributor)
raise
real_time_update, log_dict = core.handle(rt_update, trip_updates,
contributor)
# After merging trip_updates information of gtfs-rt, navitia and kirin database, if there is no new information
# destinated to navitia, update real_time_update with status = 'KO' and a proper error message.
if not real_time_update.trip_updates and real_time_update.status == 'OK':
real_time_update.status = 'KO'
real_time_update.error = 'No new information destinated to navitia for this gtfs-rt ' \
'with timestamp: {}'.format(proto.header.timestamp)
logging.getLogger(__name__).error(
'No new information destinated to navitia for this gtfs-rt '
'with timestamp: {}'.format(proto.header.timestamp))
model.db.session.add(rt_update)
model.db.session.commit()
duration = (datetime.datetime.utcnow() - start_datetime).total_seconds()
log_dict.update({
'duration':
duration,
'input_timestamp':
datetime.datetime.utcfromtimestamp(proto.header.timestamp)
})
record_call('Simple feed publication', **log_dict)
logging.getLogger(__name__).info('Simple feed publication', extra=log_dict)
def process_post(self, input_raw, contributor_type, is_new_complete=False):
# create a raw rt_update obj, save the raw_input into the db
rt_update = make_rt_update(input_raw,
contributor_type,
contributor=self.contributor)
start_datetime = datetime.utcnow()
try:
# assuming UTF-8 encoding for all input
rt_update.raw_data = rt_update.raw_data.encode('utf-8')
# raw_input is interpreted
trip_updates = self.builder(self.navitia_wrapper,
self.contributor).build(rt_update)
record_call('OK', contributor=self.contributor)
except KirinException as e:
rt_update.status = 'KO'
rt_update.error = e.data['error']
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=self.contributor)
raise
except Exception as e:
rt_update.status = 'KO'
rt_update.error = e.message
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=self.contributor)
raise
_, log_dict = core.handle(rt_update,
trip_updates,
self.contributor,
is_new_complete=is_new_complete)
duration = (datetime.utcnow() - start_datetime).total_seconds()
log_dict.update({'duration': duration})
record_call('Simple feed publication', **log_dict)
logging.getLogger(__name__).info('Simple feed publication',
extra=log_dict)
return 'OK', 200
def post(self):
raw_xml = get_ire(flask.globals.request)
# create a raw ire obj, save the raw_xml into the db
rt_update = make_rt_update(raw_xml,
'ire',
contributor=self.contributor)
start_datetime = datetime.utcnow()
try:
# assuming UTF-8 encoding for all ire input
rt_update.raw_data = rt_update.raw_data.encode('utf-8')
# raw_xml is interpreted
trip_updates = KirinModelBuilder(self.navitia_wrapper,
self.contributor).build(rt_update)
record_call('OK', contributor=self.contributor)
except KirinException as e:
rt_update.status = 'KO'
rt_update.error = e.data['error']
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=self.contributor)
raise
except Exception as e:
rt_update.status = 'KO'
rt_update.error = e.message
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=self.contributor)
raise
_, log_dict = core.handle(rt_update, trip_updates,
current_app.config['CONTRIBUTOR'])
duration = (datetime.utcnow() - start_datetime).total_seconds()
log_dict.update({'duration': duration})
record_call('Simple feed publication', **log_dict)
logging.getLogger(__name__).info('Simple feed publication',
extra=log_dict)
return 'OK', 200
def wrap_build(builder, input_raw):
"""
Function wrapping the processing of realtime information of an external feed
This manages errors/logger/newrelic
:param builder: the KirinModelBuilder to be called (must inherit from abstract_builder.AbstractKirinModelBuilder)
:param input_raw: the feed to process
"""
contributor = builder.contributor
start_datetime = datetime.datetime.utcnow()
rt_update = None
log_dict = {"contributor": contributor.id}
record_custom_parameter("contributor", contributor.id)
status = "OK"
log_dict.update({"input_feed_size": sys.getsizeof(input_raw)})
try:
# create a raw rt_update obj, save the raw_input into the db
rt_update, rtu_log_dict = builder.build_rt_update(input_raw)
log_dict.update(rtu_log_dict)
# raw_input is interpreted
trip_updates, tu_log_dict = builder.build_trip_updates(rt_update)
log_dict.update(tu_log_dict)
# finally confront to previously existing information (base_schedule, previous real-time)
_, handler_log_dict = handle(builder, rt_update, trip_updates)
log_dict.update(handler_log_dict)
except Exception as e:
status = "warning" if is_only_warning_exception(e) else "failure"
allow_reprocess = is_reprocess_allowed(e)
if rt_update is not None:
error = e.data["error"] if (isinstance(e, KirinException)
and "error" in e.data) else str(e)
set_rtu_status_ko(rt_update,
error,
is_reprocess_same_data_allowed=allow_reprocess)
db_commit(rt_update)
else:
# rt_update is not built, make sure reprocess is allowed
allow_reprocess_same_data(contributor.id)
log_dict.update({"exc_summary": str(e), "reason": str(e)})
record_custom_parameter(
"reason", str(e)) # using __str__() here to have complete details
# Re-raise all exceptions (as flask manages exceptions for output)
# try/except mechanism must wrap calls to this wrap_build() function (auto. for flask)
# See CONTRIBUTING.md for newrelic's error filtering
raise
finally:
if rt_update is not None and rt_update.status == "pending":
rt_update.status = "OK" if (status == "OK") else "KO"
db_commit(rt_update)
log_dict.update({
"duration":
(datetime.datetime.utcnow() - start_datetime).total_seconds()
})
record_call("kirin_feed_process_info", status, **log_dict)
if status == "OK":
logging.getLogger(__name__).info(status, extra=log_dict)
elif status == "warning":
logging.getLogger(__name__).warning(status, extra=log_dict)
else:
logging.getLogger(__name__).error(status, extra=log_dict)
def wrap_build(builder, input_raw):
"""
Function wrapping the processing of realtime information of an external feed
This manages errors/logger/newrelic
:param builder: the KirinModelBuilder to be called
:param input_raw: the feed to process
"""
contributor = builder.contributor
start_datetime = datetime.utcnow()
rt_update = None
log_dict = {"contributor": contributor.id}
status = "OK"
try:
# create a raw rt_update obj, save the raw_input into the db
rt_update, rtu_log_dict = builder.build_rt_update(input_raw)
log_dict.update(rtu_log_dict)
# raw_input is interpreted
trip_updates, tu_log_dict = builder.build_trip_updates(rt_update)
log_dict.update(tu_log_dict)
# finally confront to previously existing information (base_schedule, previous real-time)
_, handler_log_dict = core.handle(rt_update, trip_updates,
contributor.id,
builder.is_new_complete)
log_dict.update(handler_log_dict)
except Exception as e:
status = "failure"
allow_reprocess = True
if is_invalid_input_exception(e):
status = "warning" # Kirin did his job correctly if the input is invalid and rejected
allow_reprocess = False # reprocess is useless if input is invalid
if rt_update is not None:
error = e.data["error"] if (isinstance(e, KirinException)
and "error" in e.data) else e.message
set_rtu_status_ko(rt_update,
error,
is_reprocess_same_data_allowed=allow_reprocess)
model.db.session.add(rt_update)
model.db.session.commit()
else:
# rt_update is not built, make sure reprocess is allowed
allow_reprocess_same_data(contributor.id)
log_dict.update({"exc_summary": six.text_type(e), "reason": e})
record_custom_parameter(
"reason", e) # using __str__() here to have complete details
raise # filters later for APM (auto.)
finally:
log_dict.update(
{"duration": (datetime.utcnow() - start_datetime).total_seconds()})
record_call(status, **log_dict)
if status == "OK":
logging.getLogger(__name__).info(status, extra=log_dict)
elif status == "warning":
logging.getLogger(__name__).warning(status, extra=log_dict)
else:
logging.getLogger(__name__).error(status, extra=log_dict)
def _on_request(self, message):
log = logging.getLogger(__name__)
status = "OK"
log_dict = {}
start_datetime = datetime.utcnow()
try:
task = task_pb2.Task()
try:
# `body` is a string, but we need binary type for
# `ParseFromString()` to work. It seems to work.
body = bytes(message.payload, encoding="utf-8")
task.ParseFromString(body)
except DecodeError as e:
log.warning("invalid protobuf: {}".format(str(e)))
return
log.info("Getting a full feed publication request",
extra={"task": task})
if task.action != task_pb2.LOAD_REALTIME or not task.load_realtime:
return
begin_date = None
end_date = None
if hasattr(task.load_realtime,
"begin_date") and task.load_realtime.begin_date:
begin_date = str_to_date(task.load_realtime.begin_date)
if hasattr(task.load_realtime,
"end_date") and task.load_realtime.end_date:
end_date = str_to_date(task.load_realtime.end_date)
feed = convert_to_gtfsrt(
TripUpdate.find_by_contributor_period(
task.load_realtime.contributors, begin_date, end_date),
gtfs_realtime_pb2.FeedHeader.FULL_DATASET,
)
feed_str = feed.SerializeToString()
log_dict.update({
"contributors": task.load_realtime.contributors,
"routing_key": task.load_realtime.queue_name,
"output_trip_update_count": len(feed.entity),
"output_feed_size": sys.getsizeof(feed_str),
})
log.info(
"Starting of full feed publication {}, {}".format(
len(feed_str), task),
extra={
"size": len(feed_str),
"task": task
},
)
record_custom_parameter("contributors",
task.load_realtime.contributors)
# http://docs.celeryproject.org/projects/kombu/en/latest/userguide/producers.html#bypassing-routing-by-using-the-anon-exchange
self.producer.publish(
feed_str,
routing_key=task.load_realtime.queue_name,
retry=True,
retry_policy={
"interval_start": 0, # First retry immediately,
"interval_step": 2, # then increase by 2s for every retry.
"interval_max":
10, # but don't exceed 10s between retries.
"max_retries":
self.max_retries, # give up after 10 (by default) tries.
},
)
except Exception as e:
status = "failure"
log_dict.update({"reason": str(e)})
record_custom_parameter("reason", str(e))
finally:
duration = (datetime.utcnow() - start_datetime).total_seconds()
log_dict.update({"duration": duration})
record_call("kirin_reload_info", status, **log_dict)
log.info("End of full feed publication",
extra={
"duration": duration,
"task": task
})
db.session.remove()
return log_dict
