def poller(self):
for contributor in Contributor.find_by_connector_type(
ConnectorType.gtfs_rt.value):
config = {
"contributor": contributor.id,
"navitia_url": app.config.get("NAVITIA_URL"),
"token": contributor.navitia_token,
"coverage": contributor.navitia_coverage,
"feed_url": contributor.feed_url,
"retrieval_interval": contributor.retrieval_interval,
"timeout": app.config.get("GTFS_RT_TIMEOUT", 1),
} # WARNING: cleanup might be done here, but be cautious when removing.
# Keep in mind that during deployment, queue_workers from version n and n+1 are present
# (so no remove before both version stop using what's removed).
gtfs_poller.delay(config)
for contributor in Contributor.find_by_connector_type(
ConnectorType.siri_et_lite_idfm.value):
config = {
"contributor": contributor.id,
"navitia_url": app.config.get("NAVITIA_URL"),
"token": contributor.navitia_token,
"coverage": contributor.navitia_coverage,
"feed_url": contributor.feed_url,
"retrieval_interval": contributor.retrieval_interval,
"timeout": app.config.get("SIRI_ET_LITE_IDFM_TIMEOUT", 1),
} # WARNING: cleanup might be done here, but be cautious when removing.
# Keep in mind that during deployment, queue_workers from version n and n+1 are present
# (so no remove before both version stop using what's removed).
siri_et_lite_idfm_poller.delay(config)
def post(self, contributor_id):
if contributor_id is None or not contributor_id.strip():
abort(400, message="Contributor's id is missing")
contributor = Contributor.query_existing().filter_by(id=contributor_id).first()
if not contributor:
abort(404, message="Contributor '{}' not found".format(contributor_id))
args = flask.globals.request.args
data = flask.globals.request.data
if not data:
raise InvalidArguments("no data provided")
source_format = None
if "data_format" in args:
source_format = args["data_format"].strip().lower()
connector_type = ConnectorType(contributor.connector_type)
builder_cls = builder_cls_per_connector_type[connector_type]
try:
data = builder_cls.convert_feed(data, source_format)
except Exception as e:
raise InvalidArguments(str(e))
wrap_build(builder_cls(contributor), data)
return {"message": "'{}' feed processed".format(connector_type.value)}, 200
def test_contributor_creation():
with app.app_context():
contrib = Contributor("realtime.george", "idf", ConnectorType.cots.value)
db_commit(contrib)
assert contrib.id == "realtime.george"
assert contrib.navitia_coverage == "idf"
assert contrib.connector_type == ConnectorType.cots.value
contrib_with_same_id = Contributor("realtime.george", "another-coverage", ConnectorType.cots.value)
with pytest.raises(IntegrityError):
"""
Adding a second contributor with the same id should fail
"""
db_commit(contrib_with_same_id)
def launch_piv_worker(pg_docker_fixture):
from kirin import app
from tests.integration.conftest import PIV_CONTRIBUTOR_ID
with app.app_context():
# re-init the db by overriding the db_url
reinit_flask_db(pg_docker_fixture)
contributor = Contributor.find_one_by_connector_type_and_id(
ConnectorType.piv.value, PIV_CONTRIBUTOR_ID)
with PivWorker(contributor) as worker:
worker.run()
def process(self) -> bool:
contributors = Contributor.find_by_connector_type(
self.connector_type.value, force_refresh=True)
contributor = contributors[0] if len(contributors) > 0 else None
if not contributor:
self.logger.warning("no contributor for {}".format(
self.connector_type.value))
return True # Wait before next check that contributor exists
with self.worker_class(contributor) as worker:
self.logger.info("launching the {0} worker for '{1}'".format(
self.connector_type.value, contributor.id))
worker.run()
return False # No wait when run() was interrupted quietly (because a config changed)
def launch_siri_et_xml_tn_worker(pg_docker_fixture):
from kirin import app
from tests.integration.conftest import SIRI_ET_XML_TN_CONTRIBUTOR_DB_ID
with app.app_context():
# re-init the db by overriding the db_url
reinit_flask_db(pg_docker_fixture)
contributor = Contributor.find_one_by_connector_type_and_id(
ConnectorType.siri_et_xml_tn.value,
SIRI_ET_XML_TN_CONTRIBUTOR_DB_ID)
with SiriEtXmlTnWorker(contributor) as worker:
worker.run()
def has_contributor_been_updated(self):
"""
Get a contributor by its ID and the associated connector type
"""
# SQLAlchemy is not querying the DB for read (uses cache instead),
# unless we specifically force refresh.
contributor = Contributor.find_one_by_connector_type_and_id(
self.connector_type.value,
self.builder.contributor.id,
force_refresh=True)
return (not contributor or contributor.broker_url != self.broker_url
or contributor.navitia_coverage != self.navitia_coverage
or contributor.navitia_token != self.navitia_token
or contributor.exchange_name != self.exchange.name
or contributor.queue_name != self.queue.name)