def _complete_delete_operation(system_id):
feed_ids = set()
with dbconnection.inline_unit_of_work():
system = systemqueries.get_by_id(system_id)
for feed in system.feeds:
feed_ids.add(feed.id)
for feed_id in feed_ids:
_delete_feed(system_id, feed_id)
with dbconnection.inline_unit_of_work():
systemqueries.delete_by_id(system_id)
def _delete_feed(system_id, feed_id):
logger.info("Deleting feed {}/{}".format(system_id, feed_id))
updatemanager.execute_feed_update(
updatemanager.create_feed_flush(system_id, feed_id)
)
with dbconnection.inline_unit_of_work():
feedqueries.delete_in_system_by_id(system_id, feed_id)
def _calculate_content_hash(context: _UpdateContext):
m = hashlib.md5()
m.update(context.content)
context.feed_update.content_hash = m.hexdigest()
with dbconnection.inline_unit_of_work():
previous_hash = feedqueries.get_last_successful_update_hash(
context.feed_update.feed.pk)
if previous_hash is not None and previous_hash == context.feed_update.content_hash:
context.feed_update.status = models.FeedUpdate.Status.SUCCESS
context.feed_update.result = models.FeedUpdate.Result.NOT_NEEDED
def _initialize_update_context(feed_update_pk, content) -> _UpdateContext:
with dbconnection.inline_unit_of_work() as session:
feed_update = feedqueries.get_update_by_pk(feed_update_pk)
feed = feed_update.feed
feed_update.status = feed_update.Status.IN_PROGRESS
# We need to flush the session to persist the status change.
session.flush()
session.expunge(feed_update)
session.expunge(feed)
return _UpdateContext(feed_update=feed_update, content=content)
def _import(context: _UpdateContext):
feed_update = context.feed_update
with dbconnection.inline_unit_of_work() as session:
stats = import_.run_import(feed_update.pk, context.parser)
feed_update.num_added_entities = stats.num_added()
feed_update.num_updated_entities = stats.num_updated()
feed_update.num_deleted_entities = stats.num_deleted()
feed_update.num_parsed_entities = -1
feed_update.status = models.FeedUpdate.Status.SUCCESS
feed_update.result = models.FeedUpdate.Result.UPDATED
session.merge(feed_update)
return stats
def delete_by_id(system_id, error_if_not_exists=True, sync=True):
"""
Delete a transit system
"""
with dbconnection.inline_unit_of_work():
system = systemqueries.get_by_id(system_id)
if system is not None:
system.status = models.System.SystemStatus.DELETING
if not sync:
system.id = system.id + "_deleting_" + str(uuid.uuid4())
system_id = system.id
elif error_if_not_exists:
raise exceptions.IdNotFoundError
else:
return
client.refresh_tasks()
sync_to_function = {
True: _complete_delete_operation,
False: _complete_delete_operation_async.delay,
}
sync_to_function[sync](system_id)
def db_session(test_db):
with dbconnection.inline_unit_of_work() as session:
yield session
session.rollback()
def set_auto_update_enabled(system_id, auto_update):
with dbconnection.inline_unit_of_work():
response = systemqueries.set_auto_update_enabled(system_id, auto_update)
client.refresh_tasks()
return response
def execute_feed_update(
feed_update_pk,
content=None
) -> typing.Tuple[models.FeedUpdate, typing.Optional[Exception]]:
"""
Execute a feed update with logging and timing.
For a description of what feed updates involve consult the module docs.
"""
context = _initialize_update_context(feed_update_pk, content)
if context.feed_update.update_type == models.FeedUpdate.Type.FLUSH:
actions = _FLUSH_UPDATE_ACTIONS
else:
actions = _REGULAR_UPDATE_ACTIONS
stats = None
exception = None
for action in actions:
try:
stats = action(context)
except Exception as e:
logger.exception("Exception encountered during update")
exception_type_to_outcome = getattr(
action, "__exception_type_to_outcome__", {})
for exception_type, (status,
result) in exception_type_to_outcome.items():
if isinstance(e, exception_type):
context.feed_update.status = status
context.feed_update.result = result
context.feed_update.result_message = str(
traceback.format_exc())
exception = e
break
if context.feed_update.result_message is None:
context.feed_update.status = models.FeedUpdate.Status.FAILURE
context.feed_update.result = models.FeedUpdate.Result.UNEXPECTED_ERROR
context.feed_update.result_message = str(
traceback.format_exc())
exception = e
# If the action or one of its error handlers marked a status on the update, we
# need to finish right now.
if context.feed_update.status != models.FeedUpdate.Status.IN_PROGRESS:
break
context.feed_update.total_duration = time.time() - context.start_time
context.feed_update.completed_at = datetime.datetime.utcnow()
with dbconnection.inline_unit_of_work() as session:
session.merge(context.feed_update)
logger.info("{:7} / {:13} {:.2f} seconds / feed pk = {}.".format(
context.feed_update.status.name,
context.feed_update.result.name,
context.feed_update.total_duration,
context.feed_update.feed_pk,
))
client.feed_update_callback(
context.feed_update.feed_pk,
context.feed_update.status,
context.feed_update.result,
stats.entity_type_to_num_in_db() if stats is not None else {},
)
return context.feed_update, exception