def _build_feeds_from_config(feeds_config) -> Iterator[models.Feed]:
for id_, config in feeds_config.items():
feed = models.Feed()
feed.id = id_
feed.built_in_parser = config[systemconfigreader.PARSER].get(
systemconfigreader.BUILT_IN
)
feed.custom_parser = config[systemconfigreader.PARSER].get(
systemconfigreader.CUSTOM
)
parser_options = config[systemconfigreader.PARSER].get(
systemconfigreader.OPTIONS
)
feed.parser_options = (
json.dumps(parser_options, indent=2) if parser_options is not None else None
)
feed.url = config[systemconfigreader.HTTP][systemconfigreader.URL]
feed.headers = json.dumps(
dict(config[systemconfigreader.HTTP][systemconfigreader.HEADERS]), indent=2
)
feed.http_timeout = config[systemconfigreader.HTTP].get(
systemconfigreader.TIMEOUT
)
feed.auto_update_enabled = config[systemconfigreader.AUTO_UPDATE][
systemconfigreader.ENABLED
]
feed.auto_update_period = config[systemconfigreader.AUTO_UPDATE][
systemconfigreader.PERIOD
]
feed.required_for_install = config[systemconfigreader.REQUIRED_FOR_INSTALL]
yield feed
def test_create_feed_update(
monkeypatch,
inline_unit_of_work,
manager_function,
feed_exists,
expected_result,
expected_type,
):
if feed_exists:
feed = models.Feed(id=FEED_ID)
else:
feed = None
monkeypatch.setattr(feedqueries, "get_in_system_by_id",
lambda *args, **kwargs: feed)
def flush():
nonlocal feed
feed.updates[0].pk = FEED_UPDATE_PK
inline_unit_of_work.flush.side_effect = flush
actual_result = manager_function(SYSTEM_ID, FEED_ID)
assert actual_result == expected_result
if actual_result is None:
return
feed_update = feed.updates[0]
assert feed_update.status == models.FeedUpdate.Status.SCHEDULED
assert feed_update.update_type == expected_type
def test_execute_feed_update(
monkeypatch,
inline_unit_of_work,
custom_parser,
feed_content,
previous_content,
expected_status,
expected_explanation,
):
system = models.System(id=SYSTEM_ID)
feed = models.Feed(id=FEED_ID,
system=system,
custom_parser=custom_parser,
url=URL,
headers="{}")
feed_update = models.FeedUpdate(feed=feed)
response = mock.MagicMock()
if feed_content is None:
response.raise_for_status.side_effect = requests.exceptions.RequestException(
)
else:
response.content = feed_content
def get(*args, **kwargs):
return response
monkeypatch.setattr(requests, "get", get)
def get_update_by_pk(feed_update_pk):
return feed_update
def get_last_successful_update(*args, **kwargs):
if previous_content is None:
return None
m = hashlib.md5()
m.update(previous_content)
return m.hexdigest()
monkeypatch.setattr(feedqueries, "get_update_by_pk", get_update_by_pk)
monkeypatch.setattr(feedqueries, "get_last_successful_update_hash",
get_last_successful_update)
monkeypatch.setattr(import_, "run_import", lambda: (0, 0, 0))
feed_update, _ = updatemanager.execute_feed_update(1)
assert feed_update.status == expected_status
assert feed_update.result == expected_explanation
def test_delete__regular_case(mock_systemdam, monkeypatch):
create_feed_flush = mock.MagicMock()
monkeypatch.setattr(updatemanager, "create_feed_flush", create_feed_flush)
monkeypatch.setattr(updatemanager, "execute_feed_update", mock.MagicMock())
delete_in_system_by_id = mock.MagicMock()
monkeypatch.setattr(feedqueries, "delete_in_system_by_id",
delete_in_system_by_id)
system = models.System(id=SYSTEM_ID)
system.feeds = [models.Feed(id=FEED_ID_1)]
mock_systemdam.create(system)
systemservice.delete_by_id(SYSTEM_ID)
assert mock_systemdam.get_by_id(SYSTEM_ID) is None
create_feed_flush.assert_called_once_with(SYSTEM_ID, FEED_ID_1)
delete_in_system_by_id.assert_called_once_with(SYSTEM_ID, FEED_ID_1)
def test_execute_feed_update__success_or_sync_error(inline_unit_of_work,
monkeypatch, sync_error,
expected_status,
expected_explanation):
system = models.System(id=SYSTEM_ID)
feed = models.Feed(id=FEED_ID,
system=system,
custom_parser="custom_parser",
url=URL,
headers="{}")
feed_update = models.FeedUpdate(feed=feed)
response = mock.MagicMock()
response.content = b"a"
monkeypatch.setattr(requests, "get", lambda *args, **kwargs: response)
monkeypatch.setattr(feedqueries, "get_update_by_pk",
lambda *args: feed_update)
monkeypatch.setattr(feedqueries, "get_last_successful_update_hash",
lambda *args: None)
class Parser(parse.TransiterParser):
def load_content(self, content: bytes):
pass
monkeypatch.setattr(updatemanager, "_get_parser", lambda *args: Parser())
def sync_func(feed_update_pk, entities):
if sync_error:
raise ValueError
return importdriver.ImportStats()
monkeypatch.setattr(import_, "run_import", sync_func)
feed_update, _ = updatemanager.execute_feed_update(1)
assert feed_update.status == expected_status
assert feed_update.result == expected_explanation
def test_save_feed_configuration(monkeypatch, session_factory):
session = session_factory()
monkeypatch.setattr(dbconnection, "get_session", lambda: session)
monkeypatch.setattr(
genericqueries,
"get_id_to_pk_map",
lambda *args: {
FEED_ID_1: FEED_PK_1,
FEED_ID_2: FEED_PK_2
},
)
system = models.System()
feeds_config = {
FEED_ID_2: {
"parser": {
"built_in": "GTFS_STATIC"
},
"http": {
"url": "https://demo.transiter.dev",
"headers": {},
"timeout": 40
},
"auto_update": {
"period": None,
"enabled": False
},
"required_for_install": True,
},
FEED_ID_3: {
"parser": {
"custom": "a:b"
},
"http": {
"url": "https://nytimes.com",
"headers": {
"key": "value"
}
},
"auto_update": {
"period": 5,
"enabled": True
},
"required_for_install": False,
},
}
expected_feed_1 = models.Feed(
pk=FEED_PK_2,
id=FEED_ID_2,
built_in_parser="GTFS_STATIC",
custom_parser=None,
url="https://demo.transiter.dev",
headers="{}",
http_timeout=40,
parser_options=None,
auto_update_enabled=False,
auto_update_period=None,
required_for_install=True,
)
expected_feed_2 = models.Feed(
id=FEED_ID_3,
built_in_parser=None,
custom_parser="a:b",
url="https://nytimes.com",
headers=json.dumps({"key": "value"}, indent=2),
parser_options=None,
auto_update_period=5,
auto_update_enabled=True,
required_for_install=False,
)
feed_ids_to_update, feed_ids_to_delete = systemservice._save_feed_configuration(
system, feeds_config)
assert feed_ids_to_update == [FEED_ID_2]
assert feed_ids_to_delete == [FEED_ID_1]
assert [expected_feed_1, expected_feed_2] == session.merged
def feed_1_1(add_model, system_1):
return add_model(
models.Feed(pk=301,
id="302",
system=system_1,
auto_update_enabled=True))
def feed_2_1(add_model, system_2):
return add_model(
models.Feed(pk=311,
id="312",
system=system_2,
auto_update_enabled=True))
def feed_1_2(add_model, system_1):
return add_model(
models.Feed(pk=303,
id="304",
system=system_1,
auto_update_enabled=False))
def feed_2(add_model, system_1):
return add_model(
models.Feed(system=system_1, id="feed_2", auto_update_enabled=False))
