def _on_request(self, message):
log = logging.getLogger(__name__)
try:
task = task_pb2.Task()
try:
# `body` is of unicode type, but we need str type for
# `ParseFromString()` to work. It seems to work.
# Maybe kombu estimate that, without any information,
# the body should be something as json, and thus a
# unicode string. On the c++ side, I didn't manage to
# find a way to give a content-type or something like
# that.
body = str(message.payload)
task.ParseFromString(body)
except DecodeError as e:
log.warn('invalid protobuf: {}'.format(str(e)))
return
log.info('Getting a full feed publication request', extra={'task': task})
if task.action != task_pb2.LOAD_REALTIME or not task.load_realtime:
return
start_datetime = datetime.utcnow()
begin_date = None
end_date = None
if hasattr(task.load_realtime, "begin_date"):
if task.load_realtime.begin_date:
begin_date = str_to_date(task.load_realtime.begin_date)
if hasattr(task.load_realtime, "end_date"):
if task.load_realtime.end_date:
end_date = str_to_date(task.load_realtime.end_date)
feed = convert_to_gtfsrt(TripUpdate.find_by_contributor_period(task.load_realtime.contributors,
begin_date,
end_date),
gtfs_realtime_pb2.FeedHeader.FULL_DATASET)
feed_str = feed.SerializeToString()
log.info('Starting of full feed publication {}, {}'.format(len(feed_str), task), extra={'size': len(feed_str), 'task': task})
# http://docs.celeryproject.org/projects/kombu/en/latest/userguide/producers.html#bypassing-routing-by-using-the-anon-exchange
self.producer.publish(feed_str,
routing_key=task.load_realtime.queue_name,
retry=True,
retry_policy={
'interval_start': 0, # First retry immediately,
'interval_step': 2, # then increase by 2s for every retry.
'interval_max': 10, # but don't exceed 10s between retries.
'max_retries': self.max_retries, # give up after 10 (by default) tries.
})
duration = (datetime.utcnow() - start_datetime).total_seconds()
log.info('End of full feed publication', extra={'duration': duration, 'task': task})
record_call('Full feed publication', size=len(feed_str), routing_key=task.load_realtime.queue_name,
duration=duration, trip_update_count=len(feed.entity),
contributor=task.load_realtime.contributors)
finally:
db.session.remove()
def callback(body, message):
try:
task = task_pb2.Task()
try:
# `body` is of unicode type, but we need str type for
# `ParseFromString()` to work. It seems to work.
# Maybe kombu estimate that, without any information,
# the body should be something as json, and thus a
# unicode string. On the c++ side, I didn't manage to
# find a way to give a content-type or something like
# that.
body = str(body)
task.ParseFromString(body)
except DecodeError as e:
log.warn('invalid protobuf: {}'.format(str(e)))
return
log.info('getting a request: {}'.format(task))
if task.action != task_pb2.LOAD_REALTIME or not task.load_realtime:
return
begin_date = None
end_date = None
if hasattr(task.load_realtime, "begin_date"):
if task.load_realtime.begin_date:
begin_date = str_to_date(task.load_realtime.begin_date)
if hasattr(task.load_realtime, "end_date"):
if task.load_realtime.end_date:
end_date = str_to_date(task.load_realtime.end_date)
feed = convert_to_gtfsrt(TripUpdate.find_by_contributor_period(task.load_realtime.contributors,
begin_date,
end_date),
gtfs_realtime_pb2.FeedHeader.FULL_DATASET)
with self._get_producer() as producer:
log.info('Publishing full feed...')
producer.publish(feed.SerializeToString(), routing_key=task.load_realtime.queue_name)
log.info('Full feed published.')
finally:
db.session.remove()
def test_find_activate():
with app.app_context():
create_real_time_update(
"70866ce8-0638-4fa1-8556-1ddfa22d09d3",
COTS_CONTRIBUTOR_ID,
ConnectorType.cots.value,
"70866ce8-0638-4fa1-8556-1ddfa22d09d3",
"vj1",
datetime.date(2015, 9, 8),
)
create_real_time_update(
"70866ce8-0638-4fa1-8556-1ddfa22d09d4",
COTS_CONTRIBUTOR_ID,
ConnectorType.cots.value,
"70866ce8-0638-4fa1-8556-1ddfa22d09d4",
"vj2",
datetime.date(2015, 9, 10),
)
create_real_time_update(
"70866ce8-0638-4fa1-8556-1ddfa22d09d5",
COTS_CONTRIBUTOR_ID,
ConnectorType.cots.value,
"70866ce8-0638-4fa1-8556-1ddfa22d09d5",
"vj3",
datetime.date(2015, 9, 12),
)
create_real_time_update(
"70866ce8-0638-4fa1-8556-1ddfa22d09d6",
GTFS_CONTRIBUTOR_ID,
ConnectorType.cots.value,
"70866ce8-0638-4fa1-8556-1ddfa22d09d6",
"vj4",
datetime.date(2015, 9, 12),
)
db.session.commit()
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date 20150906 | | |
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 6))
assert len(rtu) == 3
assert rtu[0].vj_id == "70866ce8-0638-4fa1-8556-1ddfa22d09d3"
assert rtu[1].vj_id == "70866ce8-0638-4fa1-8556-1ddfa22d09d4"
assert rtu[2].vj_id == "70866ce8-0638-4fa1-8556-1ddfa22d09d5"
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date 20150908 | |
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 8))
assert len(rtu) == 3
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | 20150909 | |
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 9))
assert len(rtu) == 2
assert rtu[0].vj_id == "70866ce8-0638-4fa1-8556-1ddfa22d09d4"
assert rtu[1].vj_id == "70866ce8-0638-4fa1-8556-1ddfa22d09d5"
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | 20150910 |
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 10))
assert len(rtu) == 2
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | | 20150911 |
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 11))
assert len(rtu) == 1
assert rtu[0].vj_id == "70866ce8-0638-4fa1-8556-1ddfa22d09d5"
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | | 20150912
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 12))
assert len(rtu) == 1
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | | | 20150913
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 13))
assert len(rtu) == 0
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |---------| | | |
20150905 20150906
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 5),
datetime.date(2015, 9, 6))
assert len(rtu) == 0
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |--------------------| | |
20150905 20150908
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 5),
datetime.date(2015, 9, 8))
assert len(rtu) == 1
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |-------------------------| | |
20150905 20150909
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 5),
datetime.date(2015, 9, 9))
assert len(rtu) == 1
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |----------------------------------------------------| |
20150905 20150910
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 5),
datetime.date(2015, 9, 10))
assert len(rtu) == 2
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |----------------------------------------------------------------| |
20150905 20150911
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 5),
datetime.date(2015, 9, 11))
assert len(rtu) == 2
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |----------------------------------------------------------------------------|
20150905 20150912
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 5),
datetime.date(2015, 9, 12))
assert len(rtu) == 3
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |--------------------------------------------------------------------------------------|
20150905 20150914
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 5),
datetime.date(2015, 9, 14))
assert len(rtu) == 3
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |------------------------------------------------------------------|
20150908 20150914
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 8),
datetime.date(2015, 9, 14))
assert len(rtu) == 3
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |----------------------------------------------------------------|
20150909 20150914
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 9),
datetime.date(2015, 9, 14))
assert len(rtu) == 2
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |-----------------------------------|
20150910 20150914
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 10),
datetime.date(2015, 9, 14))
assert len(rtu) == 2
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |--------------------------------|
20150911 20150914
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 11),
datetime.date(2015, 9, 14))
assert len(rtu) == 1
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |-----------|
20150912 20150914
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 12),
datetime.date(2015, 9, 14))
assert len(rtu) == 1
"""
contributor COTS_CONTRIBUTOR
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |---------|
20150913 20150914
"""
rtu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 13),
datetime.date(2015, 9, 14))
assert len(rtu) == 0
rtu = TripUpdate.find_by_contributor_period([GTFS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 6))
assert len(rtu) == 1
rtu = TripUpdate.find_by_contributor_period([GTFS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 12))
assert len(rtu) == 1
rtu = TripUpdate.find_by_contributor_period(
[COTS_CONTRIBUTOR_ID, GTFS_CONTRIBUTOR_ID],
datetime.date(2015, 9, 12))
assert len(rtu) == 2
def test_find_activate():
with app.app_context():
create_rt_update_and_trip_update(
"70866ce8-0638-4fa1-8556-1ddfa22d09d3",
COTS_CONTRIBUTOR_ID,
"70866ce8-0638-4fa1-8556-1ddfa22d09d3",
"vj1",
datetime.date(2015, 9, 8),
)
create_rt_update_and_trip_update(
"70866ce8-0638-4fa1-8556-1ddfa22d09d4",
COTS_CONTRIBUTOR_ID,
"70866ce8-0638-4fa1-8556-1ddfa22d09d4",
"vj2",
datetime.date(2015, 9, 10),
)
create_rt_update_and_trip_update(
"70866ce8-0638-4fa1-8556-1ddfa22d09d5",
COTS_CONTRIBUTOR_ID,
"70866ce8-0638-4fa1-8556-1ddfa22d09d5",
"vj3",
datetime.date(2015, 9, 12),
)
create_rt_update_and_trip_update(
"70866ce8-0638-4fa1-8556-1ddfa22d09d6",
GTFS_CONTRIBUTOR_ID,
"70866ce8-0638-4fa1-8556-1ddfa22d09d6",
"vj4",
datetime.date(2015, 9, 12),
)
db.session.commit()
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date 20150906 | | |
"""
tu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 6))
expected_result = [
"70866ce8-0638-4fa1-8556-1ddfa22d09d3",
"70866ce8-0638-4fa1-8556-1ddfa22d09d4",
"70866ce8-0638-4fa1-8556-1ddfa22d09d5",
]
assert len(tu) == 3
assert sorted([t.vj_id for t in tu]) == expected_result
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date 20150908 | |
"""
tu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 8))
assert len(tu) == 3
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | 20150909 | |
"""
tu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 9))
expected_result = ["70866ce8-0638-4fa1-8556-1ddfa22d09d4", "70866ce8-0638-4fa1-8556-1ddfa22d09d5"]
assert len(tu) == 2
assert sorted([t.vj_id for t in tu]) == expected_result
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | 20150910 |
"""
tu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 10))
assert len(tu) == 2
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | | 20150911 |
"""
tu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 11))
assert len(tu) == 1
assert tu[0].vj_id == "70866ce8-0638-4fa1-8556-1ddfa22d09d5"
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | | 20150912
"""
tu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 12))
assert len(tu) == 1
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | | | 20150913
"""
tu = TripUpdate.find_by_contributor_period([COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 13))
assert len(tu) == 0
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |---------| | | |
20150905 20150906
"""
tu = TripUpdate.find_by_contributor_period(
[COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 5), datetime.date(2015, 9, 6)
)
assert len(tu) == 0
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |--------------------| | |
20150905 20150908
"""
tu = TripUpdate.find_by_contributor_period(
[COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 5), datetime.date(2015, 9, 8)
)
assert len(tu) == 0
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |-------------------------| | |
20150905 20150909
"""
tu = TripUpdate.find_by_contributor_period(
[COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 5), datetime.date(2015, 9, 9)
)
assert len(tu) == 1
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |----------------------------------------------------| |
20150905 20150910
"""
tu = TripUpdate.find_by_contributor_period(
[COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 5), datetime.date(2015, 9, 10)
)
assert len(tu) == 1
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |----------------------------------------------------------------| |
20150905 20150911
"""
tu = TripUpdate.find_by_contributor_period(
[COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 5), datetime.date(2015, 9, 11)
)
assert len(tu) == 2
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |----------------------------------------------------------------------------|
20150905 20150912
"""
tu = TripUpdate.find_by_contributor_period(
[COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 5), datetime.date(2015, 9, 12)
)
assert len(tu) == 2
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |--------------------------------------------------------------------------------------|
20150905 20150914
"""
tu = TripUpdate.find_by_contributor_period(
[COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 5), datetime.date(2015, 9, 14)
)
assert len(tu) == 3
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |------------------------------------------------------------------|
20150908 20150914
"""
tu = TripUpdate.find_by_contributor_period(
[COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 8), datetime.date(2015, 9, 14)
)
assert len(tu) == 3
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |----------------------------------------------------------------|
20150909 20150914
"""
tu = TripUpdate.find_by_contributor_period(
[COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 9), datetime.date(2015, 9, 14)
)
assert len(tu) == 2
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |-----------------------------------|
20150910 20150914
"""
tu = TripUpdate.find_by_contributor_period(
[COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 10), datetime.date(2015, 9, 14)
)
assert len(tu) == 2
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |--------------------------------|
20150911 20150914
"""
tu = TripUpdate.find_by_contributor_period(
[COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 11), datetime.date(2015, 9, 14)
)
assert len(tu) == 1
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |-----------|
20150912 20150914
"""
tu = TripUpdate.find_by_contributor_period(
[COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 12), datetime.date(2015, 9, 14)
)
assert len(tu) == 1
"""
contributor COTS_CONTRIBUTOR_ID
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |---------|
20150913 20150914
"""
tu = TripUpdate.find_by_contributor_period(
[COTS_CONTRIBUTOR_ID], datetime.date(2015, 9, 13), datetime.date(2015, 9, 14)
)
assert len(tu) == 0
tu = TripUpdate.find_by_contributor_period([GTFS_CONTRIBUTOR_ID], datetime.date(2015, 9, 6))
assert len(tu) == 1
tu = TripUpdate.find_by_contributor_period([GTFS_CONTRIBUTOR_ID], datetime.date(2015, 9, 12))
assert len(tu) == 1
tu = TripUpdate.find_by_contributor_period(
[COTS_CONTRIBUTOR_ID, GTFS_CONTRIBUTOR_ID], datetime.date(2015, 9, 12)
)
assert len(tu) == 2
def test_find_activate():
with app.app_context():
create_real_time_update('70866ce8-0638-4fa1-8556-1ddfa22d09d3', 'C1', 'ire',
'70866ce8-0638-4fa1-8556-1ddfa22d09d3', 'vj1', datetime.date(2015, 9, 8))
create_real_time_update('70866ce8-0638-4fa1-8556-1ddfa22d09d4', 'C1', 'ire',
'70866ce8-0638-4fa1-8556-1ddfa22d09d4', 'vj2', datetime.date(2015, 9, 10))
create_real_time_update('70866ce8-0638-4fa1-8556-1ddfa22d09d5', 'C1', 'ire',
'70866ce8-0638-4fa1-8556-1ddfa22d09d5', 'vj3', datetime.date(2015, 9, 12))
create_real_time_update('70866ce8-0638-4fa1-8556-1ddfa22d09d6', 'C2', 'ire',
'70866ce8-0638-4fa1-8556-1ddfa22d09d6', 'vj4', datetime.date(2015, 9, 12))
db.session.commit()
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date 20150906 | | |
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 6))
assert len(rtu) == 3
assert rtu[0].vj_id == '70866ce8-0638-4fa1-8556-1ddfa22d09d3'
assert rtu[1].vj_id == '70866ce8-0638-4fa1-8556-1ddfa22d09d4'
assert rtu[2].vj_id == '70866ce8-0638-4fa1-8556-1ddfa22d09d5'
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date 20150908 | |
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 8))
assert len(rtu) == 3
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | 20150909 | |
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 9))
assert len(rtu) == 2
assert rtu[0].vj_id == '70866ce8-0638-4fa1-8556-1ddfa22d09d4'
assert rtu[1].vj_id == '70866ce8-0638-4fa1-8556-1ddfa22d09d5'
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | 20150910 |
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 10))
assert len(rtu) == 2
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | | 20150911 |
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 11))
assert len(rtu) == 1
assert rtu[0].vj_id == '70866ce8-0638-4fa1-8556-1ddfa22d09d5'
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | | 20150912
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 12))
assert len(rtu) == 1
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | | | 20150913
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 13))
assert len(rtu) == 0
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |---------| | | |
20150905 20150906
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 05), datetime.date(2015, 9, 06))
assert len(rtu) == 0
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |--------------------| | |
20150905 20150908
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 05), datetime.date(2015, 9, 8))
assert len(rtu) == 1
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |-------------------------| | |
20150905 20150909
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 05), datetime.date(2015, 9, 9))
assert len(rtu) == 1
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |----------------------------------------------------| |
20150905 20150910
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 05), datetime.date(2015, 9, 10))
assert len(rtu) == 2
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |----------------------------------------------------------------| |
20150905 20150911
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 05), datetime.date(2015, 9, 11))
assert len(rtu) == 2
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |----------------------------------------------------------------------------|
20150905 20150912
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 05), datetime.date(2015, 9, 12))
assert len(rtu) == 3
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |--------------------------------------------------------------------------------------|
20150905 20150914
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 05), datetime.date(2015, 9, 14))
assert len(rtu) == 3
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |------------------------------------------------------------------|
20150908 20150914
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 8), datetime.date(2015, 9, 14))
assert len(rtu) == 3
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |----------------------------------------------------------------|
20150909 20150914
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 9), datetime.date(2015, 9, 14))
assert len(rtu) == 2
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |-----------------------------------|
20150910 20150914
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 10), datetime.date(2015, 9, 14))
assert len(rtu) == 2
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |--------------------------------|
20150911 20150914
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 11), datetime.date(2015, 9, 14))
assert len(rtu) == 1
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |-----------|
20150912 20150914
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 12), datetime.date(2015, 9, 14))
assert len(rtu) == 1
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |---------|
20150913 20150914
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 13), datetime.date(2015, 9, 14))
assert len(rtu) == 0
rtu = TripUpdate.find_by_contributor_period(['C2'], datetime.date(2015, 9, 6))
assert len(rtu) == 1
rtu = TripUpdate.find_by_contributor_period(['C2'], datetime.date(2015, 9, 12))
assert len(rtu) == 1
rtu = TripUpdate.find_by_contributor_period(['C1', 'C2'], datetime.date(2015, 9, 12))
assert len(rtu) == 2
def test_find_activate():
with app.app_context():
create_real_time_update('70866ce8-0638-4fa1-8556-1ddfa22d09d3', 'C1', 'ire',
'70866ce8-0638-4fa1-8556-1ddfa22d09d3', 'vj1', datetime.date(2015, 9, 8))
create_real_time_update('70866ce8-0638-4fa1-8556-1ddfa22d09d4', 'C1', 'ire',
'70866ce8-0638-4fa1-8556-1ddfa22d09d4', 'vj2', datetime.date(2015, 9, 10))
create_real_time_update('70866ce8-0638-4fa1-8556-1ddfa22d09d5', 'C1', 'ire',
'70866ce8-0638-4fa1-8556-1ddfa22d09d5', 'vj3', datetime.date(2015, 9, 12))
create_real_time_update('70866ce8-0638-4fa1-8556-1ddfa22d09d6', 'C2', 'ire',
'70866ce8-0638-4fa1-8556-1ddfa22d09d6', 'vj4', datetime.date(2015, 9, 12))
db.session.commit()
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date 20150906 | | |
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 6))
assert len(rtu) == 3
assert rtu[0].vj_id == '70866ce8-0638-4fa1-8556-1ddfa22d09d3'
assert rtu[1].vj_id == '70866ce8-0638-4fa1-8556-1ddfa22d09d4'
assert rtu[2].vj_id == '70866ce8-0638-4fa1-8556-1ddfa22d09d5'
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date 20150908 | |
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 8))
assert len(rtu) == 3
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | 20150909 | |
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 9))
assert len(rtu) == 2
assert rtu[0].vj_id == '70866ce8-0638-4fa1-8556-1ddfa22d09d4'
assert rtu[1].vj_id == '70866ce8-0638-4fa1-8556-1ddfa22d09d5'
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | 20150910 |
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 10))
assert len(rtu) == 2
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | | 20150911 |
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 11))
assert len(rtu) == 1
assert rtu[0].vj_id == '70866ce8-0638-4fa1-8556-1ddfa22d09d5'
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | | 20150912
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 12))
assert len(rtu) == 1
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request date | | | 20150913
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 13))
assert len(rtu) == 0
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |---------| | | |
20150905 20150906
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 05), datetime.date(2015, 9, 06))
assert len(rtu) == 0
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |--------------------| | |
20150905 20150908
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 05), datetime.date(2015, 9, 8))
assert len(rtu) == 1
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |-------------------------| | |
20150905 20150909
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 05), datetime.date(2015, 9, 9))
assert len(rtu) == 1
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |----------------------------------------------------| |
20150905 20150910
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 05), datetime.date(2015, 9, 10))
assert len(rtu) == 2
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |----------------------------------------------------------------| |
20150905 20150911
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 05), datetime.date(2015, 9, 11))
assert len(rtu) == 2
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |----------------------------------------------------------------------------|
20150905 20150912
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 05), datetime.date(2015, 9, 12))
assert len(rtu) == 3
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |--------------------------------------------------------------------------------------|
20150905 20150914
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 05), datetime.date(2015, 9, 14))
assert len(rtu) == 3
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |------------------------------------------------------------------|
20150908 20150914
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 8), datetime.date(2015, 9, 14))
assert len(rtu) == 3
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |----------------------------------------------------------------|
20150909 20150914
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 9), datetime.date(2015, 9, 14))
assert len(rtu) == 2
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |-----------------------------------|
20150910 20150914
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 10), datetime.date(2015, 9, 14))
assert len(rtu) == 2
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |--------------------------------|
20150911 20150914
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 11), datetime.date(2015, 9, 14))
assert len(rtu) == 1
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |-----------|
20150912 20150914
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 12), datetime.date(2015, 9, 14))
assert len(rtu) == 1
"""
contributor C1
VehicleJourney vj1
Circulation date 20150908 20150910 20150912
| | |
request interval date |---------|
20150913 20150914
"""
rtu = TripUpdate.find_by_contributor_period(['C1'], datetime.date(2015, 9, 13), datetime.date(2015, 9, 14))
assert len(rtu) == 0
rtu = TripUpdate.find_by_contributor_period(['C2'], datetime.date(2015, 9, 6))
assert len(rtu) == 1
rtu = TripUpdate.find_by_contributor_period(['C2'], datetime.date(2015, 9, 12))
assert len(rtu) == 1
rtu = TripUpdate.find_by_contributor_period(['C1', 'C2'], datetime.date(2015, 9, 12))
assert len(rtu) == 2
def _on_request(self, message):
log = logging.getLogger(__name__)
status = "OK"
log_dict = {}
start_datetime = datetime.utcnow()
try:
task = task_pb2.Task()
try:
# `body` is a string, but we need binary type for
# `ParseFromString()` to work. It seems to work.
body = bytes(message.payload, encoding="utf-8")
task.ParseFromString(body)
except DecodeError as e:
log.warning("invalid protobuf: {}".format(str(e)))
return
log.info("Getting a full feed publication request",
extra={"task": task})
if task.action != task_pb2.LOAD_REALTIME or not task.load_realtime:
return
begin_date = None
end_date = None
if hasattr(task.load_realtime,
"begin_date") and task.load_realtime.begin_date:
begin_date = str_to_date(task.load_realtime.begin_date)
if hasattr(task.load_realtime,
"end_date") and task.load_realtime.end_date:
end_date = str_to_date(task.load_realtime.end_date)
feed = convert_to_gtfsrt(
TripUpdate.find_by_contributor_period(
task.load_realtime.contributors, begin_date, end_date),
gtfs_realtime_pb2.FeedHeader.FULL_DATASET,
)
feed_str = feed.SerializeToString()
log_dict.update({
"contributors": task.load_realtime.contributors,
"routing_key": task.load_realtime.queue_name,
"output_trip_update_count": len(feed.entity),
"output_feed_size": sys.getsizeof(feed_str),
})
log.info(
"Starting of full feed publication {}, {}".format(
len(feed_str), task),
extra={
"size": len(feed_str),
"task": task
},
)
record_custom_parameter("contributors",
task.load_realtime.contributors)
# http://docs.celeryproject.org/projects/kombu/en/latest/userguide/producers.html#bypassing-routing-by-using-the-anon-exchange
self.producer.publish(
feed_str,
routing_key=task.load_realtime.queue_name,
retry=True,
retry_policy={
"interval_start": 0, # First retry immediately,
"interval_step": 2, # then increase by 2s for every retry.
"interval_max":
10, # but don't exceed 10s between retries.
"max_retries":
self.max_retries, # give up after 10 (by default) tries.
},
)
except Exception as e:
status = "failure"
log_dict.update({"reason": str(e)})
record_custom_parameter("reason", str(e))
finally:
duration = (datetime.utcnow() - start_datetime).total_seconds()
log_dict.update({"duration": duration})
record_call("kirin_reload_info", status, **log_dict)
log.info("End of full feed publication",
extra={
"duration": duration,
"task": task
})
db.session.remove()
return log_dict
