def test_socrata_smoke(domain, dataset_id, local_engine_empty): # This relies on the Socrata API being available, but good to smoke test some popular datasets # to make sure the mounting works end-to-end. try: mount_socrata( "socrata_mount", None, None, None, None, domain=domain, tables={"data": dataset_id} ) result = local_engine_empty.run_sql("SELECT * FROM socrata_mount.data LIMIT 10") assert len(result) == 10 finally: local_engine_empty.delete_schema("socrata_mount")
def test_socrata_mounting_slug(local_engine_empty): with open(os.path.join(INGESTION_RESOURCES, "socrata/find_datasets.json"), "r") as f: socrata_meta = json.load(f) socrata = MagicMock(spec=Socrata) socrata.datasets.return_value = socrata_meta with mock.patch("sodapy.Socrata", return_value=socrata): mount_socrata( "test/pg_mount", None, None, None, None, "example.com", None, "some_token", ) assert local_engine_empty.get_all_tables("test/pg_mount") == [ "current_employee_names_salaries_and_position_xzkq_xp2w" ]
def test_socrata_mounting_error(): socrata = MagicMock(spec=Socrata) socrata.datasets.side_effect = Exception("Unknown response format: text/html; charset=utf-8") with mock.patch("sodapy.Socrata", return_value=socrata): with pytest.raises(RepositoryNotFoundError): mount_socrata( "test/pg_mount", None, None, None, None, "example.com", {"some_table": "xzkq-xp2w"}, "some_token", )
def test_socrata_mounting_missing_tables(): with open(os.path.join(INGESTION_RESOURCES, "socrata/find_datasets.json"), "r") as f: socrata_meta = json.load(f) socrata = MagicMock(spec=Socrata) socrata.datasets.return_value = socrata_meta with mock.patch("sodapy.Socrata", return_value=socrata): with pytest.raises(ValueError) as e: mount_socrata( "test/pg_mount", None, None, None, None, "example.com", {"some_table": "wrong_id"}, "some_token", ) assert "Some Socrata tables couldn't be found! Missing tables: xzkq-xp2w" in str(e.value)
def test_socrata_mounting(local_engine_empty): with open(os.path.join(INGESTION_RESOURCES, "socrata/find_datasets.json"), "r") as f: socrata_meta = json.load(f) socrata = MagicMock(spec=Socrata) socrata.datasets.return_value = socrata_meta with mock.patch("sodapy.Socrata", return_value=socrata): mount_socrata( "test/pg_mount", None, None, None, None, "example.com", {"some_table": "xzkq-xp2w"}, "some_token", ) assert local_engine_empty.get_full_table_schema("test/pg_mount", "some_table") == [ TableColumn( ordinal=1, name=":id", pg_type="text", is_pk=False, comment="Socrata column ID" ), TableColumn( ordinal=2, name="full_or_part_time", pg_type="text", is_pk=False, comment="Whether the employee was employed full- (F) or part-time (P).", ), TableColumn( ordinal=3, name="hourly_rate", pg_type="numeric", is_pk=False, comment=mock.ANY ), TableColumn( ordinal=4, name="salary_or_hourly", pg_type="text", is_pk=False, comment=mock.ANY ), TableColumn( ordinal=5, name="job_titles", pg_type="text", is_pk=False, comment="Title of employee at the time when the data was updated.", ), TableColumn( ordinal=6, name="typical_hours", pg_type="numeric", is_pk=False, comment=mock.ANY ), TableColumn( ordinal=7, name="annual_salary", pg_type="numeric", is_pk=False, comment=mock.ANY ), TableColumn( ordinal=8, name=_long_name_col_sg, pg_type="text", is_pk=False, comment=mock.ANY ), TableColumn( ordinal=9, name="department", pg_type="text", is_pk=False, comment="Department where employee worked.", ), ] assert local_engine_empty.run_sql( "SELECT option_value FROM information_schema.foreign_table_options " "WHERE foreign_table_name = 'some_table' " "AND foreign_table_schema = 'test/pg_mount' " "AND option_name = 'column_map'" ) == [(f'{{"{_long_name_col_sg}": "{_long_name_col}"}}',)]